{"as_of":"2026-08-18T01:48:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:119a1adae48cfb322e33e251e6fd3ba17d6fa62fdfdda65acf1852bfd9bb2ba3","coverage":[{"denominator":65,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":65,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-08T16:31:38.707684Z","state":"measured"},{"denominator":71,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":71,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-17T06:30:58.91139+00:00","state":"measured"},{"denominator":6,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":6,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T19:32:36.028070Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"pith","source_observed_at":"2026-08-06T22:55:22.701914Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.06193","snapshot_observed_at":"2026-08-07T15:02:22.560982Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.16590","last_updated":"2025-09-04T04:06:40Z","snapshot_observed_at":"2026-08-16T05:48:22.641412Z","submitted_at":"2025-05-22T12:26:53Z","title":"Larger Is Not Always Better: Exploring Small Open-source Language Models in Logging Statement Generation","version":3},"reference_index":75,"source":"pdf_text","source_observed_at":"2026-08-07T15:02:22.560982Z"},"links":{"cited_paper":"/paper/2502.06193","citing_paper":"/paper/2505.16590"},"observation_digest":"sha256:6517cbaf1be65ae61723e774841adb7f0d174350a6dec3d70ef67890ec741082","observation_id":"b0bf9b93-84a3-4696-8c9a-dc533fb2cb45","resolution":{"observed_at":"2026-08-07T15:02:22.560982Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.06193","snapshot_observed_at":"2026-08-07T12:13:31.407675Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.00296","last_updated":"2025-05-30T22:58:35Z","snapshot_observed_at":"2026-08-17T15:53:37.350666Z","submitted_at":"2025-05-30T22:58:35Z","title":"CRScore++: Reinforcement Learning with Verifiable Tool and AI Feedback for Code Review","version":1},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-08-07T12:13:31.407675Z"},"links":{"cited_paper":"/paper/2502.06193","citing_paper":"/paper/2506.00296"},"observation_digest":"sha256:aa5a60d9e19905a6c00ff82929b32b4057a95837b05ae9d3b6ff267b5462e7df","observation_id":"ac31a24c-c423-4aa8-bf6f-cc421c3367c4","resolution":{"observed_at":"2026-08-07T12:13:31.407675Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.06193","snapshot_observed_at":"2026-08-07T00:40:17.022076Z","title":"Can LLMs replace human evaluators? An empirical study of LLM-as-a-Judge in software engineering,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.13171","last_updated":"2025-06-16T07:34:28Z","snapshot_observed_at":"2026-08-16T23:37:00.579707Z","submitted_at":"2025-06-16T07:34:28Z","title":"Querying Large Automotive Software Models: Agentic vs. Direct LLM Approaches","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-07T00:40:17.022076Z"},"links":{"cited_paper":"/paper/2502.06193","citing_paper":"/paper/2506.13171"},"observation_digest":"sha256:15a3f0acab89eea29b0f8ac6e38022211e07ac9ddd7c5d126522ab35bf9af30f","observation_id":"1a0d2f36-7f99-4b90-863c-425c93e235a5","resolution":{"observed_at":"2026-08-07T00:40:17.022076Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.06193","snapshot_observed_at":"2026-08-15T19:32:36.028070Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.16440","last_updated":"2025-06-19T16:18:53Z","snapshot_observed_at":"2026-08-16T19:09:31.176165Z","submitted_at":"2025-06-19T16:18:53Z","title":"Evaluating the Use of LLMs for Documentation to Code Traceability","version":1},"reference_index":81,"source":"pdf_text","source_observed_at":"2026-08-15T19:32:36.028070Z"},"links":{"cited_paper":"/paper/2502.06193","citing_paper":"/paper/2506.16440"},"observation_digest":"sha256:1d955effbf682af650ea7488e7340f9c2331c965e58eead4d60e7b17235b982d","observation_id":"a2e97899-30c0-4865-ad11-be9784b177b8","resolution":{"observed_at":"2026-08-15T19:32:36.028070Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.06193","snapshot_observed_at":"2026-08-06T23:12:14.277116Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.19185","last_updated":"2025-06-23T23:02:57Z","snapshot_observed_at":"2026-08-17T03:12:55.567276Z","submitted_at":"2025-06-23T23:02:57Z","title":"Spiritual-LLM : Gita Inspired Mental Health Therapy In the Era of LLMs","version":1},"reference_index":49,"source":"arxiv_source","source_observed_at":"2026-08-06T23:12:14.277116Z"},"links":{"cited_paper":"/paper/2502.06193","citing_paper":"/paper/2506.19185"},"observation_digest":"sha256:bc02cf88f954560c469814f4d8e183f6d470fcaa45bfd7c6cbc3d26fcc29df67","observation_id":"f548eb6d-8b09-4ea9-83a0-367b254dba3d","resolution":{"observed_at":"2026-08-06T23:12:14.277116Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"cited_work":{"arxiv_id":"2502.06193","doi":null,"metadata_source":"pith","pith_arxiv_id":"2502.06193","snapshot_observed_at":"2026-08-06T22:55:22.701914Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","venue":"cs.SE","work_id":"392ccf14-3ed4-4384-bdc5-c2cd39b5afda","year":2025},"citing_paper":{"arxiv_id":"2506.20558","last_updated":"2025-06-25T15:56:07Z","snapshot_observed_at":"2026-08-15T05:22:57.539979Z","submitted_at":"2025-06-25T15:56:07Z","title":"CCISolver: End-to-End Detection and Repair of Method-Level Code-Comment Inconsistency","version":1},"reference_index":77,"source":"pdf_text","source_observed_at":"2026-08-06T22:55:22.623455Z"},"links":{"cited_paper":"/paper/2502.06193","citing_paper":"/paper/2506.20558"},"observation_digest":"sha256:f8552fba79ed3dea949e857e614789e85adb997cb1b60f0bf8fc85e7bd0a6314","observation_id":"9462f634-0751-458c-984c-38e6b37bddfb","resolution":{"observed_at":"2026-08-06T22:55:22.740075Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2502.06193/citation-record","integrity":"/paper/2502.06193/integrity","json":"/paper/2502.06193/citation-record.json","paper":"/paper/2502.06193"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2108.07732","last_updated":"2021-08-16T03:57:30Z","snapshot_observed_at":"2026-08-15T17:40:38.050939Z","submitted_at":"2021-08-16T03:57:30Z","title":"Program Synthesis with Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.07732","snapshot_observed_at":"2026-08-08T16:31:38.500723Z","title":"Nye, Maarten Bosma, Henryk Michalewski, David Dohan, Ellen Jiang, Carrie J","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.500723Z"},"links":{"cited_paper":"/paper/2108.07732","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:ea8c3e8f40e495198af82ae3ba656212b8a1e61cb9d951764560a105e64be809","observation_id":"3868d53e-fb24-4bfc-83f0-d5e6420ae3bb","resolution":{"observed_at":"2026-08-08T16:31:38.500723Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.258077Z","title":null,"venue":null,"work_id":"fd689f9d-fef8-436c-b584-1f59363fa552","year":2005},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.505054Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:813a2fdd9481d4a5b3ecd4a8daf4ea3e99c67f505d02d5ff11eb3143ca1f74d1","observation_id":"e068fdfd-4cd9-4104-a28b-8c29d8280bf3","resolution":{"observed_at":"2026-08-08T16:31:39.261024Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.249929Z","title":null,"venue":null,"work_id":"da0f0c22-4c2b-4856-b73c-2a6f74b33d1e","year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.508451Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:27f4d9f7d5b385f23f92ee6ba33875425bf42de58282358259fafceb3b4e891b","observation_id":"ab8ebfde-a82c-4f38-8e82-d8dffec19534","resolution":{"observed_at":"2026-08-08T16:31:39.252552Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.242237Z","title":null,"venue":null,"work_id":"f3effce0-3bcc-4152-b77d-1a3b837cfed8","year":null},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.511407Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:09f269979444479cea15adeaf075a43e79560a769be886a51ba4901c6a76946d","observation_id":"ec301540-cd7d-4044-a9b9-4fd2a3864f9a","resolution":{"observed_at":"2026-08-08T16:31:39.244734Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.11931","last_updated":"2024-06-17T13:51:35Z","snapshot_observed_at":"2026-08-14T04:42:19.578053Z","submitted_at":"2024-06-17T13:51:35Z","title":"DeepSeek-Coder-V2: Breaking the Barrier of Closed-Source Models in Code Intelligence","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.11931","snapshot_observed_at":"2026-08-08T16:31:38.518553Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.518553Z"},"links":{"cited_paper":"/paper/2406.11931","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:45016c927c566bf5f9c7cd09e770f813443ed79dcbe5f8c226fffebdfd9b359b","observation_id":"3e2f3999-7198-438a-8654-770fbb9171b8","resolution":{"observed_at":"2026-08-08T16:31:38.518553Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.233868Z","title":null,"venue":null,"work_id":"9d89379c-697a-4ee5-a6d3-5147febe35a7","year":2019},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.522419Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:102d7ae2becfc5e99c0ec46d80eb8080e2587d98be750d517f638c529028f8cc","observation_id":"af6ca29c-4399-45d6-af62-61bd049563cd","resolution":{"observed_at":"2026-08-08T16:31:39.237275Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2308.01861","last_updated":"2023-08-14T09:07:00Z","snapshot_observed_at":"2026-08-16T15:10:59.697868Z","submitted_at":"2023-08-03T16:31:02Z","title":"ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.01861","snapshot_observed_at":"2026-08-08T16:31:38.525989Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.525989Z"},"links":{"cited_paper":"/paper/2308.01861","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:b86798c53fb49c6dfdee34596f5377ffd3b2b23d81f4003353d0aa1de4ef7152","observation_id":"a131398c-471e-415a-9e50-79f619d19e36","resolution":{"observed_at":"2026-08-08T16:31:38.525989Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.226265Z","title":null,"venue":null,"work_id":"7e2e99a7-91a4-4341-b169-4cb7c05268fc","year":2022},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.529399Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:a3db4c1afd17fb56a5e828571803ed2dff1bf84127a50ca82a7c63af6ad8f3d4","observation_id":"5bfc84ef-99ff-47b5-87f8-2f2a7df77bb0","resolution":{"observed_at":"2026-08-08T16:31:39.228688Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2409.10280","last_updated":"2024-09-16T13:43:04Z","snapshot_observed_at":"2026-08-16T13:18:11.033277Z","submitted_at":"2024-09-16T13:43:04Z","title":"ComplexCodeEval: A Benchmark for Evaluating Large Code Models on More Complex Code","version":1},"cited_work":{"arxiv_id":"2409.10280","doi":null,"metadata_source":"pith","pith_arxiv_id":"2409.10280","snapshot_observed_at":"2026-08-08T16:31:38.869159Z","title":"ComplexCodeEval: A Benchmark for Evaluating Large Code Models on More Complex Code","venue":"cs.SE","work_id":"cdb00220-68ed-4fed-82d8-30ff894e31c3","year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.532439Z"},"links":{"cited_paper":"/paper/2409.10280","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:74277eb7119b90c5945d6260e31910bedf9cc73857be5f9e98702733bd1901e5","observation_id":"e08bd5b7-87f0-468a-add7-328667f053fd","resolution":{"observed_at":"2026-08-08T16:31:38.875140Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.217023Z","title":null,"venue":null,"work_id":"3db6ba8f-076e-4180-b97d-5eb264675281","year":2020},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.537606Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:a9d9c89397122430e2ded27657e6ed364d065e355352618d52320da6a886f142","observation_id":"956a22f1-63ee-4de7-b238-c4cfbb466594","resolution":{"observed_at":"2026-08-08T16:31:39.220654Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.207768Z","title":null,"venue":null,"work_id":"cacec8f3-fa3d-44af-9d14-d484dc635955","year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.540873Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:074fe6bef2923d7547457fd8bee4a8f724bff258e8316636751601ae51eae48d","observation_id":"6c49c30b-b42b-46ba-bfbd-c64740c3c994","resolution":{"observed_at":"2026-08-08T16:31:39.210455Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.01383","last_updated":"2025-05-14T06:05:53Z","snapshot_observed_at":"2026-08-16T14:22:09.467736Z","submitted_at":"2024-02-02T13:06:35Z","title":"LLM-based NLG Evaluation: Current Status and Challenges","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.01383","snapshot_observed_at":"2026-08-08T16:31:38.543764Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.543764Z"},"links":{"cited_paper":"/paper/2402.01383","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:08ad96aae8faeac1e86e1212f35e37bdd2598507520b68fa1dad4c5892bbf679","observation_id":"10ff1f27-ce36-479c-8ab8-6103ae517494","resolution":{"observed_at":"2026-08-08T16:31:38.543764Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.198521Z","title":null,"venue":null,"work_id":"b381cb9e-4a00-4cb6-b6ac-bf1999dc72b2","year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.547576Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:a149cefffb06b9c1e96cb86cbf03df352bb81ec71b5fd1303265c9507654e24f","observation_id":"4e49bdf6-813e-4a77-b009-27f48a3c89e9","resolution":{"observed_at":"2026-08-08T16:31:39.201088Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.188779Z","title":null,"venue":null,"work_id":"52a18a23-b04e-47ee-a9ee-10fe713c30a6","year":2022},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.550842Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:52b35dc59148dfba38c4d598744fc0fcfc59cf898418f5673c542c8788a13b07","observation_id":"5ecdf1d1-5b53-41c6-9554-f8d186b0d679","resolution":{"observed_at":"2026-08-08T16:31:39.191394Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.14196","last_updated":"2024-01-26T09:23:11Z","snapshot_observed_at":"2026-08-06T22:40:28.707813Z","submitted_at":"2024-01-25T14:17:53Z","title":"DeepSeek-Coder: When the Large Language Model Meets Programming -- The Rise of Code Intelligence","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.14196","snapshot_observed_at":"2026-08-08T16:31:38.554014Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.554014Z"},"links":{"cited_paper":"/paper/2401.14196","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:96d14bd3155cff7b3da0523d3f9035f9d90a04d5d8667fe2e45379507170f3bb","observation_id":"c5afba38-8efb-4ca8-806f-7779d19d797f","resolution":{"observed_at":"2026-08-08T16:31:38.554014Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.180065Z","title":null,"venue":null,"work_id":"1d296ac7-c797-45fe-99ab-63bd6f2ec06a","year":2021},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.557110Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:92f907c4ae2c04ed9b4c8db355045ebe768a6968f2c17266469ffddcb9dc4060","observation_id":"743708d9-63f8-4b77-991f-d47bcf6ba81c","resolution":{"observed_at":"2026-08-08T16:31:39.183058Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2308.10620","last_updated":"2024-04-10T08:41:22Z","snapshot_observed_at":"2026-08-16T15:07:02.749105Z","submitted_at":"2023-08-21T10:37:49Z","title":"Large Language Models for Software Engineering: A Systematic Literature Review","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.10620","snapshot_observed_at":"2026-08-08T16:31:38.559051Z","title":"Grundy, and Haoyu Wang","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.559051Z"},"links":{"cited_paper":"/paper/2308.10620","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:1f0835ab4e55145afe3947aa7af0960624a2400e5b477e3cf7472a9d35be4d7d","observation_id":"de1a3493-6bd5-436c-b038-af8b2bd7d887","resolution":{"observed_at":"2026-08-08T16:31:38.559051Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.12186","last_updated":"2024-11-12T13:24:25Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-09-18T17:57:57Z","title":"Qwen2.5-Coder Technical Report","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.12186","snapshot_observed_at":"2026-08-08T16:31:38.561533Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.561533Z"},"links":{"cited_paper":"/paper/2409.12186","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:38c97ba216abb73ada4ab88117c40d775eb7f7d7019cd3f50d90427d8563bbed","observation_id":"49dcfa92-8cb7-4efb-9907-ca65ed383115","resolution":{"observed_at":"2026-08-08T16:31:38.561533Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1909.09436","last_updated":"2020-06-08T09:09:28Z","snapshot_observed_at":"2026-08-15T20:39:17.343191Z","submitted_at":"2019-09-20T11:52:45Z","title":"CodeSearchNet Challenge: Evaluating the State of Semantic Code Search","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1909.09436","snapshot_observed_at":"2026-08-08T16:31:38.563930Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.563930Z"},"links":{"cited_paper":"/paper/1909.09436","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:53d61c1dcc82d7c5e3f1d0b5dc9de13d20befa75ff74936d26f14e41d48d001c","observation_id":"cc199193-66b6-482e-be04-863e10348b9d","resolution":{"observed_at":"2026-08-08T16:31:38.563930Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.171366Z","title":null,"venue":null,"work_id":"66dd2a01-9abf-47d3-b86d-8fd6fa56fcf5","year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.566466Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:dceb69e8dcede87eede29d922f32f26ade44585b20ec86e80350b56aa60d89bf","observation_id":"904d21e8-0574-47e2-a38a-40349a14cc5b","resolution":{"observed_at":"2026-08-08T16:31:39.174756Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.160652Z","title":null,"venue":null,"work_id":"f60a8a78-3b58-424c-bd59-be041b5c9e46","year":null},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.568522Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:24f0c6f2b68b6a964670e1df956d25ae28798826dbf9b37edb4e93871ef6b8b2","observation_id":"9ff7e2cb-85be-4c34-8efd-055bb94520bd","resolution":{"observed_at":"2026-08-08T16:31:39.164109Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.01535","last_updated":"2024-12-04T19:23:17Z","snapshot_observed_at":"2026-08-16T13:55:51.395455Z","submitted_at":"2024-05-02T17:59:35Z","title":"Prometheus 2: An Open Source Language Model Specialized in Evaluating Other Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.01535","snapshot_observed_at":"2026-08-08T16:31:38.573942Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.573942Z"},"links":{"cited_paper":"/paper/2405.01535","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:1a31e2c1997cf467f95cf8b64d42d8bc61978550d2b839ed63bc59a378f082eb","observation_id":"683d7c4b-09d5-45cd-aac8-4830e0839131","resolution":{"observed_at":"2026-08-08T16:31:38.573942Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.151789Z","title":"Kusner, Yu Sun, Nicholas I","venue":null,"work_id":"5854c56a-532b-40c6-935d-f30c3b260d22","year":2015},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.577177Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:32170f9723bacbd9f4018ed48ba35c989fb05daeca3842eb8d81d2b1223fc248","observation_id":"1d6c4246-09cc-420c-9377-60b00d940dd2","resolution":{"observed_at":"2026-08-08T16:31:39.154619Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.143243Z","title":null,"venue":null,"work_id":"2d1fd49f-76f6-4f1b-a3b4-abc2486403af","year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.579789Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:0795bd64516d340479e2ff928a1150a4ad0f4c383a0ff97519bd379359b262e3","observation_id":"625fb6f0-0ab3-477d-bec9-7e1a3d57aae5","resolution":{"observed_at":"2026-08-08T16:31:39.145883Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.134814Z","title":null,"venue":null,"work_id":"1c827197-0bbf-447b-903b-a2e9ad3c3871","year":2020},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.583337Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:9214dcdafa1a15d8e90f20743ad583dae67827c46ed9092ca9e814a6c2f3ecd5","observation_id":"ddba5173-d099-45cd-a9b4-952d08f55a54","resolution":{"observed_at":"2026-08-08T16:31:39.137467Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.126099Z","title":null,"venue":null,"work_id":"3fc4825f-2de3-4054-9425-e2f4f3868231","year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.587061Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:df6ebd6648cbe9968f0af550199b3f899c75c5287f26a37b8ba81ba0fb4b153b","observation_id":"2d175b8f-c051-48ec-96ef-c7f3a9aab792","resolution":{"observed_at":"2026-08-08T16:31:39.129921Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.117056Z","title":null,"venue":null,"work_id":"21fb55b5-6058-434f-8071-3cabb2474b14","year":2022},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.590551Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:51195c36a86dfb6f569c494a26953db7ddb97ff164beefdb252a0d93ac0600ad","observation_id":"851d245e-b47b-4b7a-9f1e-3d80fac4e74e","resolution":{"observed_at":"2026-08-08T16:31:39.120619Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.01432","last_updated":"2024-12-09T09:39:12Z","snapshot_observed_at":"2026-08-16T14:56:29.581269Z","submitted_at":"2023-09-29T14:38:58Z","title":"Split and Merge: Aligning Position Biases in LLM-based Evaluators","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.01432","snapshot_observed_at":"2026-08-08T16:31:38.593488Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.593488Z"},"links":{"cited_paper":"/paper/2310.01432","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:5d7eb88bbec8433d4718321ebba82fa295c1ca9cb3d9a9393a2e77cadc13096e","observation_id":"dd6b44d1-7f41-4933-8eed-74cefb1c3116","resolution":{"observed_at":"2026-08-08T16:31:38.593488Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:38.596966Z","title":null,"venue":null,"work_id":null,"year":2004},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.596966Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:7f25d1a775ba19503f6193bad7461fd982c834145f5dbc795d3998ee382d209a","observation_id":"e4d2f7a9-d5c3-473b-ae23-b0814c4176df","resolution":{"observed_at":"2026-08-08T16:31:38.596966Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.103447Z","title":null,"venue":null,"work_id":"2abb4131-f50e-46d8-bc74-0f4804f3ca0f","year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.600295Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:123491f86339520fa1561367a3605c7aef90c97e086e9c7a6c7c74cbceeff1d3","observation_id":"3741a2cb-f73d-4e7b-83da-6c2bdb6ecad0","resolution":{"observed_at":"2026-08-08T16:31:39.105980Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.094361Z","title":null,"venue":null,"work_id":"79d7d9b9-8ca9-4063-98a2-07e09d317be8","year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.603057Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:77866561821e9728db22009ca95fc7b863e5981e40c77c511dfbc894ee388e4c","observation_id":"ec961d15-95b5-46f2-96e3-7adba1e94f81","resolution":{"observed_at":"2026-08-08T16:31:39.097227Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.084587Z","title":null,"venue":null,"work_id":"dfa59681-97ae-4602-8e40-402b7f8f8b73","year":2021},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.606396Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:56607098c14601e8254ff824ba256735e59488a2df77c8d2b97a8c18fb580ae8","observation_id":"54930fdc-081a-40d5-a139-25abe9c6deba","resolution":{"observed_at":"2026-08-08T16:31:39.086782Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-17T09:58:46.058102Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-08T16:31:38.610051Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.610051Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:a6135adc3609342087e8f88c6ea220987667330f6167980f0f540fb9a9858aae","observation_id":"f15b9b71-4f8a-46c1-a300-50b8cb1232da","resolution":{"observed_at":"2026-08-08T16:31:38.610051Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.077740Z","title":null,"venue":null,"work_id":"a77cff1b-ad76-471c-868d-6d00dfae6832","year":2002},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.612737Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:6cce82cb67ad53163cb2d9121f5490cd31f11dbc951f377ebe78477d5cadbc44","observation_id":"c71b889e-340d-4c67-81f3-622e4ff6a820","resolution":{"observed_at":"2026-08-08T16:31:39.080053Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.069464Z","title":null,"venue":null,"work_id":"896330f7-1848-4159-83a8-887fdfda04e6","year":2017},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.616355Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:779f0e1e20077ad45adbdccea31e91dacc80e02dbe98e67e50387d89d1a51162","observation_id":"fd462a9e-f213-4fd4-8569-a3c7508c3a0f","resolution":{"observed_at":"2026-08-08T16:31:39.072503Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:38.619860Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.619860Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:a8de307178499fdd5504de2f01fc6528e5e02d7cede5c550950cedc802bbd0b9","observation_id":"e185c660-b8b8-43b1-8383-3040f4d310a5","resolution":{"observed_at":"2026-08-08T16:31:38.619860Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.05530","last_updated":"2024-12-16T17:39:39Z","snapshot_observed_at":"2026-08-14T18:15:53.516440Z","submitted_at":"2024-03-08T18:54:20Z","title":"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.05530","snapshot_observed_at":"2026-08-08T16:31:38.623144Z","title":"Lillicrap, Jean-Baptiste Alayrac, Radu Soricut, Angeliki Lazaridou, Orhan Firat, Julian Schrittwieser, Ioannis Antonoglou, Rohan Anil, Sebastian Borgeaud, Andrew M","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.623144Z"},"links":{"cited_paper":"/paper/2403.05530","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:c3385f22985dd963fae99170882ac125d347d3059a4baeee52f7c71a465bc098","observation_id":"397d1667-4f3e-4c19-a58f-c657d54b7ae8","resolution":{"observed_at":"2026-08-08T16:31:38.623144Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.10297","last_updated":"2020-09-27T04:07:11Z","snapshot_observed_at":"2026-08-12T14:26:33.940158Z","submitted_at":"2020-09-22T03:10:49Z","title":"CodeBLEU: a Method for Automatic Evaluation of Code Synthesis","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.10297","snapshot_observed_at":"2026-08-08T16:31:38.626832Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.626832Z"},"links":{"cited_paper":"/paper/2009.10297","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:c54d2bf469aacdc393688d5681b7e1750909736dfd306b3e56db8b4fe478c2a3","observation_id":"c8e03438-6b55-474e-8f5e-5b208c04e4a8","resolution":{"observed_at":"2026-08-08T16:31:38.626832Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.12950","last_updated":"2024-01-31T19:47:26Z","snapshot_observed_at":"2026-08-13T04:25:38.282910Z","submitted_at":"2023-08-24T17:39:13Z","title":"Code Llama: Open Foundation Models for Code","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.12950","snapshot_observed_at":"2026-08-08T16:31:38.630490Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.630490Z"},"links":{"cited_paper":"/paper/2308.12950","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:e14cb4f17bdec63f46b7864ddbed6cf0eae69d3847134ffc281e33ab54e0e077","observation_id":"3062d33b-5d31-4d8e-9838-71c482ab3be6","resolution":{"observed_at":"2026-08-08T16:31:38.630490Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.10076","last_updated":"2023-10-16T05:19:02Z","snapshot_observed_at":"2026-08-16T14:51:54.448036Z","submitted_at":"2023-10-16T05:19:02Z","title":"Verbosity Bias in Preference Labeling by Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.10076","snapshot_observed_at":"2026-08-08T16:31:38.634452Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.634452Z"},"links":{"cited_paper":"/paper/2310.10076","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:c4229b699de62c10107acc74a5dabee5e67f7288bb9d38efd21e66eb573685ed","observation_id":"a74136e1-ddbe-4979-8751-a2538862a7ad","resolution":{"observed_at":"2026-08-08T16:31:38.634452Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.15963","last_updated":"2024-09-29T05:03:25Z","snapshot_observed_at":"2026-08-16T23:08:12.927804Z","submitted_at":"2024-01-29T08:47:31Z","title":"NoFunEval: Funny How Code LMs Falter on Requirements Beyond Functional Correctness","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.15963","snapshot_observed_at":"2026-08-08T16:31:38.639436Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.639436Z"},"links":{"cited_paper":"/paper/2401.15963","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:d3277d28de4da331791414ccdc8c6651f93acacf929ec45a3ca71c07fa744d9a","observation_id":"712d1f2d-f67c-4ef0-9661-92e09ac45324","resolution":{"observed_at":"2026-08-08T16:31:38.639436Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.03314","last_updated":"2024-08-06T17:35:05Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-08-06T17:35:05Z","title":"Scaling LLM Test-Time Compute Optimally can be More Effective than Scaling Model Parameters","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.03314","snapshot_observed_at":"2026-08-08T16:31:38.643324Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.643324Z"},"links":{"cited_paper":"/paper/2408.03314","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:3fe560286e1604d98db03db5981e2c82e07e29412b874a0569f014f288d555cc","observation_id":"07ce0fdd-0c28-4312-b72f-6fabc4b445cd","resolution":{"observed_at":"2026-08-08T16:31:38.643324Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.09288","last_updated":"2023-07-19T17:08:59Z","snapshot_observed_at":"2026-08-07T12:56:43.323460Z","submitted_at":"2023-07-18T14:31:57Z","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.09288","snapshot_observed_at":"2026-08-08T16:31:38.646275Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.646275Z"},"links":{"cited_paper":"/paper/2307.09288","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:088e7229278f7457e940eea430cc88363cc65d6ddbbb51855fcc8442aa7c706b","observation_id":"09a9a8ba-6507-4bc9-8474-ea8107c38b1a","resolution":{"observed_at":"2026-08-08T16:31:38.646275Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.054220Z","title":"Gomez, Lukasz Kaiser, and Illia Polosukhin","venue":null,"work_id":"e94efe87-a25d-40ee-a123-7361a135e05b","year":2017},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.649265Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:9e4f409beaf6954039b4c1827fd158480fb39cf924e0ed186a116bcf7527ec08","observation_id":"f02908e9-591c-4cd7-9609-c75c34de2fc5","resolution":{"observed_at":"2026-08-08T16:31:39.057626Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.03304","last_updated":"2024-10-04T03:58:14Z","snapshot_observed_at":"2026-08-16T14:54:54.816980Z","submitted_at":"2023-10-05T04:15:48Z","title":"Learning Personalized Alignment for Evaluating Open-ended Text Generation","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.03304","snapshot_observed_at":"2026-08-08T16:31:38.651288Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.651288Z"},"links":{"cited_paper":"/paper/2310.03304","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:7d281ec06e4013f38646d09c62fbeb3f4b0afafec19cd10165a3bf9404b269db","observation_id":"bb3c7809-27f8-4148-80a9-857657c5d296","resolution":{"observed_at":"2026-08-08T16:31:38.651288Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.044298Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","venue":null,"work_id":"5fb8e875-3f18-4996-9272-eaf36d068f40","year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.654100Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:cb40e3b2a2175a8f5b7fa61850c315387e8d27bcd8a8cdf3785b6bd76a5a677f","observation_id":"128287bf-556e-4083-af8e-bbf2711cf374","resolution":{"observed_at":"2026-08-08T16:31:39.047541Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.035039Z","title":"Joty, and Steven C","venue":null,"work_id":"5fc759af-d424-4ca4-95cb-9f92a1fcf590","year":2021},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.658148Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:52d75fd812d7462aa9bc0e67a587114eada47069b65395eba5a033c960bd0a20","observation_id":"5fe2802b-a071-4324-b8af-ed5358711612","resolution":{"observed_at":"2026-08-08T16:31:39.038228Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.017391Z","title":null,"venue":null,"work_id":"2bfb8d5a-a99b-4a7c-9608-285e605bca03","year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.664718Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:09639975fc199cc2683aa21372aebffb7e20cce3e97479024bf6a489cedb6127","observation_id":"0ea2c352-6e87-4bf7-a24c-75d043bee362","resolution":{"observed_at":"2026-08-08T16:31:39.020527Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.008299Z","title":"Chi, Tatsunori Hashimoto, Oriol Vinyals, Percy Liang, Jeff Dean, and William Fedus","venue":null,"work_id":"f482c13f-9d74-42c4-ad58-0773b6d06b7f","year":2022},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.668006Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:be867eacdecd52b5c5e72b1cbeea5fcd084f5d4f75e226e7386e93025cf10618","observation_id":"9cc4ecad-3d9e-4a3d-886c-59118aa037a1","resolution":{"observed_at":"2026-08-08T16:31:39.011635Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:39.025922Z","title":null,"venue":null,"work_id":"83b03e6d-c2c0-4bcb-ba6b-b1f39759e3ca","year":null},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.661459Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:01ca14ff71d222de479ac116179b85e046f8e55914c6c44f685df15e8bf5ce57","observation_id":"77714668-0593-4495-9211-c02e02f2474f","resolution":{"observed_at":"2026-08-08T16:31:39.028850Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:38.994217Z","title":null,"venue":null,"work_id":"c8f0c48f-5a63-4eca-946b-8e79c203f0f2","year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.674609Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:80305cc024f9d59028456026b7eace7fad5d4cf9802eba9fdd4289a5958a9f89","observation_id":"a45db5d7-53cc-4652-a2bd-cbfa9e67f7f0","resolution":{"observed_at":"2026-08-08T16:31:38.997099Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.09032","last_updated":"2024-12-27T05:13:23Z","snapshot_observed_at":"2026-08-16T14:09:59.223904Z","submitted_at":"2024-03-14T01:51:35Z","title":"CodeUltraFeedback: An LLM-as-a-Judge Dataset for Aligning Large Language Models to Coding Preferences","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.09032","snapshot_observed_at":"2026-08-08T16:31:38.677847Z","title":"Sahraoui","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.677847Z"},"links":{"cited_paper":"/paper/2403.09032","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:67cff305dc04bef6e085fefccd0fd93a4fa325a5b7817153945bbe2ca3827f1f","observation_id":"6b31838a-276d-4e0b-a872-5ebca17882ce","resolution":{"observed_at":"2026-08-08T16:31:38.677847Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:38.671920Z","title":"Chi, Quoc V","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.671920Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:8e16238e91180784f462d65e90199b32c26337d1fef2760d83003e90e5b1a0e0","observation_id":"5af6ff84-71ef-498b-96e8-9a26d7517236","resolution":{"observed_at":"2026-08-08T16:31:38.671920Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:38.986136Z","title":null,"venue":null,"work_id":"d96208cb-5247-497e-be47-1dcb077491bf","year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.683699Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:d29749f62655ffc761272f56280d9a8c5f614f341e6c99315e72d47ca6025403","observation_id":"8e6f94c8-1105-43d0-8855-2bfdbd7709c8","resolution":{"observed_at":"2026-08-08T16:31:38.989091Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:38.975950Z","title":null,"venue":null,"work_id":"7480aa4c-7d7d-41cf-9ee1-901b2381e6dc","year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.687006Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:2ced16df2a90294bb5f6822f7a4e76734afc61f44779acdfe2425993bda83d69","observation_id":"48d51f9a-d463-42da-8277-229b8b289b23","resolution":{"observed_at":"2026-08-08T16:31:38.978907Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1910.03771","last_updated":"2020-07-14T03:42:34Z","snapshot_observed_at":"2026-07-06T08:27:58.343233Z","submitted_at":"2019-10-09T03:23:22Z","title":"HuggingFace's Transformers: State-of-the-art Natural Language Processing","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1910.03771","snapshot_observed_at":"2026-08-08T16:31:38.680884Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.680884Z"},"links":{"cited_paper":"/paper/1910.03771","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:b91eba7c12381ce56c216c6c1c630ee6cf97383f038cd78fb0caa21448892c90","observation_id":"7a245692-579b-40b4-8cdd-82ee9e57bae4","resolution":{"observed_at":"2026-08-08T16:31:38.680884Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:38.957756Z","title":null,"venue":null,"work_id":"4085d6df-0d89-4d0f-874a-b5a9b1ecc111","year":2021},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.692334Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:61cf2a753f6538461c29644bdeb1db94e368ab54041d19b3f80f3be0ad1c581e","observation_id":"20b34a92-8a61-4467-b013-4aaf9d9661ab","resolution":{"observed_at":"2026-08-08T16:31:38.961533Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:38.947939Z","title":"Weinberger, and Yoav Artzi","venue":null,"work_id":"79aec283-ecb5-4c61-90f5-12e0ef1a0707","year":2020},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.695214Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:eb5aa0e2da5df37dbb4b7a3a388eb47940a8f3bc7714ae71965a7ee4c6a6fa23","observation_id":"c50c506b-7e68-4874-9c53-550c8e21a2da","resolution":{"observed_at":"2026-08-08T16:31:38.951345Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:38.967289Z","title":null,"venue":null,"work_id":"a9cae13f-de30-401c-9c19-490f18c9227e","year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.689697Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:0a660b682dbc888bf63d854bb9b28a06c308c7ac5db0338780f14d809ef98f86","observation_id":"d623d1c9-a276-4439-9a33-7cfe21876ff0","resolution":{"observed_at":"2026-08-08T16:31:38.970306Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:38.928858Z","title":"Xing, Hao Zhang, Joseph E","venue":null,"work_id":"ab416552-73b6-4831-8433-e9d689666cbb","year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.701312Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:4f65571a10a40c3bdc23db3528e548f4baecfef8ff0012f931b2bbd7c615c293","observation_id":"10399c38-2c81-4c96-9631-8d37f115350c","resolution":{"observed_at":"2026-08-08T16:31:38.931257Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:38.920709Z","title":null,"venue":null,"work_id":"04eb3a95-b4c9-4649-b355-37c50cb7d171","year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.704472Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:30d7ee9fbb885bd527a9bb100b9cafcb07b9a87cc39a85ad5682927029cf7d2b","observation_id":"35433146-82a4-41f6-a277-f6dfd458f085","resolution":{"observed_at":"2026-08-08T16:31:38.923243Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:38.938885Z","title":"Meyer, and Steffen Eger","venue":null,"work_id":"b024fb16-ef6a-4548-9c8c-de46631eee66","year":2019},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.698437Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:13f775f47caffe4d6349a4df97dad8164cc7d78ab0ad0daf374ec359075d9de8","observation_id":"93b47eda-514b-42c9-8afa-ff8154dc4212","resolution":{"observed_at":"2026-08-08T16:31:38.941241Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:31:38.910969Z","title":null,"venue":null,"work_id":"1e236b56-61f7-4f18-beda-28f786750875","year":2023},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":65,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.707684Z"},"links":{"citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:38791dc03a0fa77af6a260972b2ceb3164203eb7e8eebc3413a979f72183b216","observation_id":"909abd9b-028c-42f7-92fe-b16f2773e1c3","resolution":{"observed_at":"2026-08-08T16:31:38.914535Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T11:58:24.516369Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-08T16:31:38.514729Z","title":"CoRR abs/2107.03374 (2021)","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.514729Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:36bb41bb27936fdc14ca9eff65d2ae7d3d275eac38ee18ade414a2e12edf161c","observation_id":"21e5fe0a-f968-4da0-aae1-d8e8ec1fde5a","resolution":{"observed_at":"2026-08-08T16:31:38.514729Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.04088","last_updated":"2024-01-08T18:47:34Z","snapshot_observed_at":"2026-08-13T19:43:49.936776Z","submitted_at":"2024-01-08T18:47:34Z","title":"Mixtral of Experts","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.04088","snapshot_observed_at":"2026-08-08T16:31:38.570816Z","title":"CoRR abs/2401.04088 (2024)","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering","version":3},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-08T16:31:38.570816Z"},"links":{"cited_paper":"/paper/2401.04088","citing_paper":"/paper/2502.06193"},"observation_digest":"sha256:07def04647058f7fb786eec035ffd49b17de61f23068815e4865b554fb76abc4","observation_id":"81b21440-3ee7-4b66-a13d-09cd647d13cc","resolution":{"observed_at":"2026-08-08T16:31:38.570816Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2502.06193","last_updated":"2025-04-21T08:41:21Z","latest_version":3,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-17T15:12:55.148987Z","submitted_at":"2025-02-10T06:49:29Z","title":"Can LLMs Replace Human Evaluators? An Empirical Study of LLM-as-a-Judge in Software Engineering"},"reference_resolution":{"displayed":65,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":56,"verified_exact":1,"verified_fuzzy":8},"total_outbound_references":65},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"thesis":"As of 18 August 2026, this Paper Citation Record lists 65 of 65 outbound references and 6 inbound Pith citation observations for arXiv:2502.06193."}