{"as_of":"2026-08-14T12:57:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:741e6ffe9a4cba917df8887ef266615127cf26472442bffdad0e25a76b724e59","coverage":[{"denominator":70,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":70,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T11:59:18.434546Z","state":"measured"},{"denominator":72,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":72,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-14T06:32:32.682623+00:00","state":"measured"},{"denominator":2,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":2,"source":"paper_references, paper_reference_links","source_observed_at":"2026-06-29T10:32:47.756343Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-06-29T10:33:18.382028Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"cited_work":{"arxiv_id":"2411.17793","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2411.17793","snapshot_observed_at":"2026-06-29T10:33:18.382028Z","title":"Engineering ai judge systems.arXiv preprint arXiv:2411.17793,","venue":null,"work_id":"e330a923-5efb-4faf-bb9c-619db1109c48","year":2024},"citing_paper":{"arxiv_id":"2506.13538","last_updated":"2026-04-13T03:45:24Z","snapshot_observed_at":"2026-08-12T23:06:53.486968Z","submitted_at":"2025-06-16T14:26:37Z","title":"Model Context Protocol (MCP) at First Glance: Studying the Security and Maintainability of MCP Servers","version":5},"reference_index":82,"source":"pdf_text","source_observed_at":"2026-05-19T09:32:08.596259Z"},"links":{"cited_paper":"/paper/2411.17793","citing_paper":"/paper/2506.13538"},"observation_digest":"sha256:41583265bfd1918620e9777a95549122203b7f51a224ddd547efcf58339b18c4","observation_id":"a1f08563-e1d9-450b-b1fb-942ab696aa28","resolution":{"observed_at":"2026-05-19T09:32:15.515399Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"cited_work":{"arxiv_id":"2411.17793","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2411.17793","snapshot_observed_at":"2026-06-29T10:33:18.382028Z","title":"Engineering ai judge systems.arXiv preprint arXiv:2411.17793,","venue":null,"work_id":"e330a923-5efb-4faf-bb9c-619db1109c48","year":2024},"citing_paper":{"arxiv_id":"2606.00118","last_updated":"2026-05-27T19:42:01Z","snapshot_observed_at":"2026-08-14T03:09:36.364059Z","submitted_at":"2026-05-27T19:42:01Z","title":"An Empirical Study on Logging Evolution On Stack Overflow: Trends, Topics, and Challenges","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-06-29T10:32:47.756343Z"},"links":{"cited_paper":"/paper/2411.17793","citing_paper":"/paper/2606.00118"},"observation_digest":"sha256:0d1d9e09ee4eef7b83515590cc885c7e54d5f67b16950a2631cade26e8b558de","observation_id":"c3ec2d89-6bea-4e6b-8f3b-09f152b10195","resolution":{"observed_at":"2026-06-29T10:33:18.383835Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2411.17793/citation-record","integrity":"/paper/2411.17793/integrity","json":"/paper/2411.17793/citation-record.json","paper":"/paper/2411.17793"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.290245Z","title":"Claude 3 sonnet has become very lazy,","venue":null,"work_id":"4592b282-298e-42cd-b8c4-c7a9e0d5a9f3","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.139088Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:7a0f0a2cdd0e220026d5998daca20430c2bfeb1049957c79436be84f570fb6cc","observation_id":"e07a633d-a329-4f9a-822d-47e8a45c6cad","resolution":{"observed_at":"2026-08-12T11:59:19.294465Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.267409Z","title":"Do you guy think the cost of gpt-4 is high,","venue":null,"work_id":"2a5dd227-ec4d-4c41-8ed9-7d4bb6f2a088","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.148769Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:c4843f16e21a23d810dc543d9d5628c85f5816b288bfcc369e65a1365b69ebf9","observation_id":"0641d58c-b4d9-45ae-a544-f469463b3b76","resolution":{"observed_at":"2026-08-12T11:59:19.271435Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.255422Z","title":"Gpt-4 is crazy expensive,","venue":null,"work_id":"bb7556f8-98f9-4737-9582-e54ea74958ea","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.152711Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:789db54b288f4bab227e17810a324aea09e70786647e9a6f89cf9a8fc9e49b93","observation_id":"7e88b40b-4104-444c-a7f7-955aa0354452","resolution":{"observed_at":"2026-08-12T11:59:19.259355Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.244068Z","title":"Open llm leaderboard,","venue":null,"work_id":"fac51eaf-7f89-4ffa-9f9b-5168786f15ce","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.157331Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:3fefc8fabf169dcfb1e12ad89f2538edfa5ea0c6658913c1ffa1821679112c93","observation_id":"6e7e3ff0-5b01-49c5-8426-0c64801f4e06","resolution":{"observed_at":"2026-08-12T11:59:19.247901Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.231842Z","title":"Use agent metrics & llm judges to evaluate app performance,","venue":null,"work_id":"661e4c05-bde9-4656-ab37-9030b2a19b5c","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.161619Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:9a9f92272acdbaa95a506b28d7de1ac9e836ea7a1a6e2d03fdbb3778868d95c2","observation_id":"a62f18e3-1362-46cb-9a8a-d5e82ba916dd","resolution":{"observed_at":"2026-08-12T11:59:19.236198Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.219945Z","title":"2030 software engineering,","venue":null,"work_id":"5036765d-3fe1-4296-82c0-be458333edc5","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.166669Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:89c4cd4f579b200964fecd5bdea42ed719ffcbf46c58ccfdd297618bd8d85b50","observation_id":"2d24a91c-5b70-4273-bff3-90967b30d88b","resolution":{"observed_at":"2026-08-12T11:59:19.224133Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.206181Z","title":"The acm international conference on the foundations of software en- gineering (fse) 2024,","venue":null,"work_id":"e46c9323-2c09-408d-8feb-62d575586f79","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.170796Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:1523b58d231f6932d1846e6efc7353b8cbacb5b0cb184b4e8a72c00a9044b021","observation_id":"947c5aeb-8423-4bc7-9dda-6df89c092418","resolution":{"observed_at":"2026-08-12T11:59:19.210844Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.194119Z","title":"Fm+se summit 2024,","venue":null,"work_id":"de371124-c69f-42cc-85fd-d91d74d43ced","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.175887Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:a4727727be1af3025c99333b54f9585479bcec580f9c9051068eefea6c85ec27","observation_id":"05ca5043-7480-4d7e-8b16-34110261baea","resolution":{"observed_at":"2026-08-12T11:59:19.198220Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.182103Z","title":"Opea initiative (open platform for enterprise ai (opea),","venue":null,"work_id":"97043aaf-82da-4eea-8696-1ca375179b61","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.180239Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:b999719d012f53c583cbc4e38ce15d8dc5bd4584ed5440ec82207e6cd244b94f","observation_id":"40e1b4f1-a253-4d29-a173-96e35aede6e5","resolution":{"observed_at":"2026-08-12T11:59:19.186229Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.168415Z","title":"Re-thinking data strategy and integration for artificial intelligence: concepts, opportuni- ties, and challenges,","venue":null,"work_id":"b6c193fc-483c-4579-87fa-d5f53884aad3","year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.185399Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:6ee49d42d5ccbe048a449b289e533bafde281d92f74bffdaa97633178d1792d5","observation_id":"c3f25090-1dee-424a-9f4d-ef4075c739ff","resolution":{"observed_at":"2026-08-12T11:59:19.172791Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2212.08073","last_updated":"2022-12-15T06:19:23Z","snapshot_observed_at":"2026-08-02T04:53:58.766070Z","submitted_at":"2022-12-15T06:19:23Z","title":"Constitutional AI: Harmlessness from AI Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2212.08073","snapshot_observed_at":"2026-08-12T11:59:18.189110Z","title":"Constitutional ai: harmlessness from ai feedback,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.189110Z"},"links":{"cited_paper":"/paper/2212.08073","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:6e6ed63adf29a9e8291ddd2a0528d41fdda2dfbe22b96ed60d36e5eaf18b660b","observation_id":"8a957eb4-3f66-4202-b0f1-8923229c292d","resolution":{"observed_at":"2026-08-12T11:59:18.189110Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.03927","last_updated":"2024-02-22T12:32:24Z","snapshot_observed_at":"2026-08-13T04:25:14.624829Z","submitted_at":"2024-02-06T11:54:23Z","title":"Leak, Cheat, Repeat: Data Contamination and Evaluation Malpractices in Closed-Source LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.03927","snapshot_observed_at":"2026-08-12T11:59:18.194285Z","title":"Leak, cheat, repeat: data contamination and evaluation malpractices in closed-source llms,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.194285Z"},"links":{"cited_paper":"/paper/2402.03927","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:bc917ffa6c82387620b5a9e1792fcfc0d8fd92e3ffcb57b7219d1165980c21c6","observation_id":"c8023431-371b-4128-96f9-6d4dd42ca5d2","resolution":{"observed_at":"2026-08-12T11:59:18.194285Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.155055Z","title":"Meteor: an automatic metric for MT evalua- tion with improved correlation with human judgments,","venue":null,"work_id":"63f9d79d-853f-40c6-84ca-686c9b556ce3","year":2005},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.198230Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:17aeedd1a49dfb220256c75660c4b4913de5fdb88495b0b2bd332cb8014b1aa3","observation_id":"92cf6acd-09fd-4426-b0ef-6bf32fe48823","resolution":{"observed_at":"2026-08-12T11:59:19.159428Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.143293Z","title":"Test driven development: By example addison-wesley,","venue":null,"work_id":"c3139465-1423-4643-88d2-079f7bc3b4b3","year":2002},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.201903Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:7331b1fa01eabd9c93cd0294f557f4dc911455a3adc42b035549037b65afb8f6","observation_id":"cfb058a0-0668-4acf-87dd-305edac4a074","resolution":{"observed_at":"2026-08-12T11:59:19.147267Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.131649Z","title":"On the dangers of stochastic parrots: can language models be too big?","venue":null,"work_id":"ba4edb82-a945-4903-b584-bb04d08a92b7","year":2021},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.205397Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:1fb39b3874f7237b68a7d226a7861710d19513bbacacbecf5d368a647cb29659","observation_id":"71493e67-f94e-479d-a0cd-602bac9facd1","resolution":{"observed_at":"2026-08-12T11:59:19.135774Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2303.12712","last_updated":"2023-04-13T20:41:31Z","snapshot_observed_at":"2026-08-03T04:49:15.195814Z","submitted_at":"2023-03-22T16:51:28Z","title":"Sparks of Artificial General Intelligence: Early experiments with GPT-4","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.12712","snapshot_observed_at":"2026-08-12T11:59:18.208969Z","title":"Sparks of artificial general intelligence: early experiments with gpt-4,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.208969Z"},"links":{"cited_paper":"/paper/2303.12712","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:6298464339fee9d2d990f8a79f9083adf62cdf9364a868ed50b358add1f3443e","observation_id":"c2ac4a3a-4cff-48fd-97bf-e8fd9f8a5dd5","resolution":{"observed_at":"2026-08-12T11:59:18.208969Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.07201","last_updated":"2023-08-14T15:13:04Z","snapshot_observed_at":"2026-08-02T01:34:38.978920Z","submitted_at":"2023-08-14T15:13:04Z","title":"ChatEval: Towards Better LLM-based Evaluators through Multi-Agent Debate","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.07201","snapshot_observed_at":"2026-08-12T11:59:18.212705Z","title":"Chateval: towards better llm-based evaluators through multi-agent debate,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.212705Z"},"links":{"cited_paper":"/paper/2308.07201","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:a7633946ca559e94d8640717543f24283b79d784d751bfbb689d033d8d7d51ec","observation_id":"62e3339c-7943-452c-847f-8cbde46d5111","resolution":{"observed_at":"2026-08-12T11:59:18.212705Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.118603Z","title":"A survey on evaluation of large language models,","venue":null,"work_id":"103cb2e5-27a5-4dc5-a59b-729158cb0f94","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.217544Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:1635956b77258ca52d7487da4a4ed7440386986cece2eef791ce50309c8269ea","observation_id":"0935c594-8e41-416a-95db-9f86a061e380","resolution":{"observed_at":"2026-08-12T11:59:19.122927Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.14735","last_updated":"2025-05-11T09:23:41Z","snapshot_observed_at":"2026-08-13T05:43:18.634746Z","submitted_at":"2023-10-23T09:15:18Z","title":"Unleashing the potential of prompt engineering for large language models","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.14735","snapshot_observed_at":"2026-08-12T11:59:18.221024Z","title":"Unleashing the potential of prompt engineering in large language models: a comprehensive review,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.221024Z"},"links":{"cited_paper":"/paper/2310.14735","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:9ce9c8b4f211719c7f8567b4a4ebd45964b15643212fbe867546f041eba4d85b","observation_id":"e65b98a8-a1b8-4100-a638-188fd934f478","resolution":{"observed_at":"2026-08-12T11:59:18.221024Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.106443Z","title":"Towards training reproducible deep learning models,","venue":null,"work_id":"f9325a4b-74f4-4c80-b6a1-690d1f14c850","year":2022},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.224849Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:38672d5bfe834148e12168395f046a8c029aaa66e826c6a1e714ad442a9fb0b7","observation_id":"a970bc9c-49d3-465b-9583-2f2c3cd9b86e","resolution":{"observed_at":"2026-08-12T11:59:19.110698Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.10669","last_updated":"2024-09-26T03:16:52Z","snapshot_observed_at":"2026-08-13T04:17:35.878300Z","submitted_at":"2024-02-16T13:21:06Z","title":"Humans or LLMs as the Judge? A Study on Judgement Biases","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.10669","snapshot_observed_at":"2026-08-12T11:59:18.228507Z","title":"Humans or llms as the judge? a study on judgement biases,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.228507Z"},"links":{"cited_paper":"/paper/2402.10669","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:8ba47a1142510d12b1ce1317d68c68f4512408f23d4f45cb07272ad07933c831","observation_id":"d2fa95c7-8c31-498e-8bdf-dec0558784c8","resolution":{"observed_at":"2026-08-12T11:59:18.228507Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.09009","last_updated":"2023-10-31T16:13:44Z","snapshot_observed_at":"2026-08-14T03:29:44.245855Z","submitted_at":"2023-07-18T06:56:08Z","title":"How is ChatGPT's behavior changing over time?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.09009","snapshot_observed_at":"2026-08-12T11:59:18.233491Z","title":"How is chatgpt’s behavior changing over time?","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.233491Z"},"links":{"cited_paper":"/paper/2307.09009","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:f803da1bb2f35801e9d8b8d6e44ef59dfb20ee3c8b7e8097a80df1a3c2055150","observation_id":"c5ad56e2-3a5c-4c91-a84c-08c4ee58a9c2","resolution":{"observed_at":"2026-08-12T11:59:18.233491Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.04132","last_updated":"2024-03-07T01:22:38Z","snapshot_observed_at":"2026-08-02T17:55:33.750637Z","submitted_at":"2024-03-07T01:22:38Z","title":"Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.04132","snapshot_observed_at":"2026-08-12T11:59:18.237211Z","title":"Chatbot arena: an open platform for evaluating llms by human preference,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.237211Z"},"links":{"cited_paper":"/paper/2403.04132","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:417ebb94f6ac17490a0a0caa391e2cef515846162db50da5a2926cf3da507050","observation_id":"64e25685-2f9e-4c1b-9113-cc8e95572a23","resolution":{"observed_at":"2026-08-12T11:59:18.237211Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.278962Z","title":"Available: https://www.reddit.com/r/ClaudeAI/comments/ 1bv8ww5/claude 3 sonnet has become very lazy/","venue":null,"work_id":"7e85d1df-75fd-44df-97b5-ba311af5e62b","year":null},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.143956Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:014ec19e22aed868e11cb15c4d712ff6de327628490a9811bab9dcf95f478d65","observation_id":"2ddc37fd-3306-4aac-bcb2-a6cc389bc29f","resolution":{"observed_at":"2026-08-12T11:59:19.282971Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2110.14168","last_updated":"2021-11-18T00:23:45Z","snapshot_observed_at":"2026-08-14T02:43:01.480086Z","submitted_at":"2021-10-27T04:49:45Z","title":"Training Verifiers to Solve Math Word Problems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.14168","snapshot_observed_at":"2026-08-12T11:59:18.241194Z","title":"Training verifiers to solve math word problems,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.241194Z"},"links":{"cited_paper":"/paper/2110.14168","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:7c6b5b51ab826071825f2e2bfc12099f0201dec0ab728be1c180e6ee485f8fe2","observation_id":"10d1d72c-38ce-4f89-8809-379d44341fc4","resolution":{"observed_at":"2026-08-12T11:59:18.241194Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.093882Z","title":"Evalullm: llm assisted evaluation of generative outputs,","venue":null,"work_id":"d98933fb-0bcd-49f3-b580-fe42518b476d","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.245859Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:303d2df34d77f074c59425865f65c8e1c2318debd88cce4a555f37528cc5f2cf","observation_id":"54a73f2e-70a0-43f2-89e7-684da527c5e6","resolution":{"observed_at":"2026-08-12T11:59:19.098115Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.081950Z","title":"Qlora: ef- ficient finetuning of quantized llms,","venue":null,"work_id":"769cd384-bcab-4e72-ac32-bf6452814be5","year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.249801Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:ad856771f03bbd900700e2217c09d352cd23fe09bd323309fff5e7d94ea15b6a","observation_id":"2ab6465c-c725-4a04-b939-394da8855761","resolution":{"observed_at":"2026-08-12T11:59:19.086396Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.071374Z","title":"Fira: fine-grained graph-based code change representation for automated com- mit message generation,","venue":null,"work_id":"9692b42e-47c9-4a56-983b-256081c93f66","year":2022},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.253234Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:b34c9f2d15b246a1abf26fc83def287d9d0b22aa1235def40d20043e1062b2b8","observation_id":"aec2f2f2-46ed-4a8e-9983-d24f1b299583","resolution":{"observed_at":"2026-08-12T11:59:19.075156Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.059979Z","title":"Alpacafarm: a simulation framework for methods that learn from human feedback,","venue":null,"work_id":"8b582c22-ffbb-40c8-bacd-ed8b20487f81","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.256879Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:f0badcedf4bf80264dac74dbcd559c2d46bf937381b1b14ea8a38345423e48e6","observation_id":"2c4f6a65-7ce3-4ec6-a895-0cbd0da38f37","resolution":{"observed_at":"2026-08-12T11:59:19.063807Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.048693Z","title":"Bias and fairness in large language models: a survey,","venue":null,"work_id":"64ea2262-b710-4b08-87cc-a16d07fe501b","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.260862Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:7299577ab1751adb3c2d2ca207e6d62eb5249bcf78dc0ba2483653b37ad02d29","observation_id":"9831402a-94fc-4b4a-bb73-8d52cc27b3c7","resolution":{"observed_at":"2026-08-12T11:59:19.052740Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2205.12679","last_updated":"2023-02-26T08:14:05Z","snapshot_observed_at":"2026-08-13T15:37:22.234748Z","submitted_at":"2022-05-25T11:38:48Z","title":"Self-Guided Noise-Free Data Generation for Efficient Zero-Shot Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2205.12679","snapshot_observed_at":"2026-08-12T11:59:18.266978Z","title":"Self-guided noise- free data generation for efficient zero-shot learning,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.266978Z"},"links":{"cited_paper":"/paper/2205.12679","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:d62c928d92c3e4c35e70c1920ee7ccd9e9fdbb32819de2806036d2d48c7f758d","observation_id":"535c660a-2eb5-4e4a-8c8c-2d803839f901","resolution":{"observed_at":"2026-08-12T11:59:18.266978Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.01383","last_updated":"2025-05-14T06:05:53Z","snapshot_observed_at":"2026-08-13T04:28:25.209847Z","submitted_at":"2024-02-02T13:06:35Z","title":"LLM-based NLG Evaluation: Current Status and Challenges","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.01383","snapshot_observed_at":"2026-08-12T11:59:18.271869Z","title":"Llm-based nlg evaluation: current status and challenges,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.271869Z"},"links":{"cited_paper":"/paper/2402.01383","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:ec1c5cc6e3b2b004744bec33ff56d80f8973971be0127f70c6a41061a6904ebf","observation_id":"4944e0f9-35d0-4078-8829-e8f5a610a8e9","resolution":{"observed_at":"2026-08-12T11:59:18.271869Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.037040Z","title":"Fm+se vision 2030,","venue":null,"work_id":"53583729-09e3-4843-9016-ac6c98f39393","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.276954Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:6c998410067bb2b4993d1214996ba51069bc4630f7f9816b4a257baaed434211","observation_id":"964d0df2-5403-4e1b-8136-a64903194b90","resolution":{"observed_at":"2026-08-12T11:59:19.041370Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.025221Z","title":"Rethinking software engineering in the foundation model era: a curated catalogue of challenges in the development of trustworthy fmware,","venue":null,"work_id":"d1f1481e-bfe5-43bf-bceb-654eafca54b7","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.280771Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:a426f33c78d56bb1173b2360e73b7bbecfc01548289ca2036128798221abea26","observation_id":"80f2e391-f7a0-4f0b-9247-39c261d7d047","resolution":{"observed_at":"2026-08-12T11:59:19.029239Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2009.03300","last_updated":"2021-01-12T18:57:11Z","snapshot_observed_at":"2026-08-13T20:44:28.824685Z","submitted_at":"2020-09-07T17:59:25Z","title":"Measuring Massive Multitask Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.03300","snapshot_observed_at":"2026-08-12T11:59:18.285054Z","title":"Measuring massive multitask language understanding,","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.285054Z"},"links":{"cited_paper":"/paper/2009.03300","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:8d31d2d0e7fb619fe95e0f7a19ca7724f8b9ed25809e9a365934ee43f6bde2e5","observation_id":"38c394b4-fb5d-4586-b09d-45b121f5a5ef","resolution":{"observed_at":"2026-08-12T11:59:18.285054Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.05232","last_updated":"2024-11-19T12:42:45Z","snapshot_observed_at":"2026-07-06T16:45:07.733095Z","submitted_at":"2023-11-09T09:25:37Z","title":"A Survey on Hallucination in Large Language Models: Principles, Taxonomy, Challenges, and Open Questions","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.05232","snapshot_observed_at":"2026-08-12T11:59:18.289124Z","title":"A survey on hallucination in large language models: principles, taxonomy, challenges, and open questions,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.289124Z"},"links":{"cited_paper":"/paper/2311.05232","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:a160a8c994efc1d051877e71de67ddeffa8c9f4c31ae49d4f0589e3b87808415","observation_id":"abc6d780-04f7-4f85-bcb2-8b0d8f8b217b","resolution":{"observed_at":"2026-08-12T11:59:18.289124Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1805.00899","last_updated":"2018-10-22T17:36:07Z","snapshot_observed_at":"2026-08-02T15:33:17.783178Z","submitted_at":"2018-05-02T16:27:32Z","title":"AI safety via debate","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1805.00899","snapshot_observed_at":"2026-08-12T11:59:18.293117Z","title":"Ai safety via debate,","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.293117Z"},"links":{"cited_paper":"/paper/1805.00899","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:de5a63d423bd92e4d723630f6f35343bd8d7863927499c936858fcb05e617cc0","observation_id":"8f6c5c9f-b355-4868-9765-adf08a204dd1","resolution":{"observed_at":"2026-08-12T11:59:18.293117Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.014434Z","title":"Kejriwal, Domain-specific knowledge graph construction","venue":null,"work_id":"932d745e-16b8-424a-9a0e-71c135db553c","year":2019},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.297827Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:f76c06b75e23d6959b79f2a80530bfcaac515799378f66bfe82afd016e6acc85","observation_id":"601eb689-23ad-4590-b3ee-368c505574dd","resolution":{"observed_at":"2026-08-12T11:59:19.018229Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.04622","last_updated":"2024-07-12T16:38:12Z","snapshot_observed_at":"2026-08-13T17:36:04.704742Z","submitted_at":"2024-07-05T16:29:15Z","title":"On scalable oversight with weak LLMs judging strong LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.04622","snapshot_observed_at":"2026-08-12T11:59:18.301527Z","title":"On scalable oversight with weak llms judging strong llms,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.301527Z"},"links":{"cited_paper":"/paper/2407.04622","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:efc4ff050770fb5b6aef3b835cbb1c534ea2d6847423dc41251cc2ff6bdf2e39","observation_id":"dce9d1bd-1dd9-479a-9f89-db458e0f2469","resolution":{"observed_at":"2026-08-12T11:59:18.301527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.06782","last_updated":"2024-07-25T23:32:21Z","snapshot_observed_at":"2026-08-13T04:22:15.640251Z","submitted_at":"2024-02-09T21:05:01Z","title":"Debating with More Persuasive LLMs Leads to More Truthful Answers","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.06782","snapshot_observed_at":"2026-08-12T11:59:18.305344Z","title":"Debating with more persuasive llms leads to more truthful answers,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.305344Z"},"links":{"cited_paper":"/paper/2402.06782","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:b103c9b735f481d206d58d4140f9e4a21dfc4c6d4f0dfd87793a57991ee0006a","observation_id":"28284a8d-410a-4e89-b44b-1d1c75aff2f3","resolution":{"observed_at":"2026-08-12T11:59:18.305344Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.002278Z","title":"Dspy: compiling declarative language model calls into state-of-the-art pipelines,","venue":null,"work_id":"0f5a83d8-48a2-4036-9f44-c907cab86e01","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.309227Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:9748a03a027583c0c3775c07ea6964bb9fb2d58f6eb07c3511e180b90cd2a3dd","observation_id":"84a337a4-3562-4833-9efb-ddcd9b7d00f5","resolution":{"observed_at":"2026-08-12T11:59:19.007287Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.989500Z","title":"Software engineering for machine learning applications (semla) 2023,","venue":null,"work_id":"b8c86638-1174-401e-910b-d771b71f7f0e","year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.313972Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:9236cece0b52e60c338a1dbb674fa8d3372366d5affd517e33f0389fdf3b5116","observation_id":"f9386766-e1fd-461f-9013-e120f2ed1260","resolution":{"observed_at":"2026-08-12T11:59:18.993839Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.06452","last_updated":"2024-02-19T14:39:07Z","snapshot_observed_at":"2026-08-13T16:47:08.208696Z","submitted_at":"2023-10-10T09:25:44Z","title":"Understanding the Effects of RLHF on LLM Generalisation and Diversity","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.06452","snapshot_observed_at":"2026-08-12T11:59:18.317823Z","title":"Understanding the effects of rlhf on llm generalisation and diversity,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.317823Z"},"links":{"cited_paper":"/paper/2310.06452","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:c3fbd6e910da081a59b08ed7371dceb125205deca236793dcecbab0ed748b721","observation_id":"dac9e283-2527-47fb-8842-c2df3acbb6cc","resolution":{"observed_at":"2026-08-12T11:59:18.317823Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.976977Z","title":"On the role of knowledge graphs in explainable ai,","venue":null,"work_id":"fe06dc94-cb34-4077-a852-b1b994d615d5","year":2020},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.322382Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:9875341cfdf02f27c0bf38bfaacc72d4ad2fe1ab4ffe678bbb973046d7702b78","observation_id":"3aec817e-5fca-4029-af5c-2709ab851d84","resolution":{"observed_at":"2026-08-12T11:59:18.981263Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2305.19118","last_updated":"2024-10-09T02:41:21Z","snapshot_observed_at":"2026-08-01T16:20:31.337598Z","submitted_at":"2023-05-30T15:25:45Z","title":"Encouraging Divergent Thinking in Large Language Models through Multi-Agent Debate","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.19118","snapshot_observed_at":"2026-08-12T11:59:18.326017Z","title":"Encouraging divergent thinking in large language models through multi-agent debate,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.326017Z"},"links":{"cited_paper":"/paper/2305.19118","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:cc79e83a23727af8f721eb936aecad56d205737e66ebf0a424a6d685a50b297a","observation_id":"2104f4a8-8d00-479d-aa47-f19387188b8c","resolution":{"observed_at":"2026-08-12T11:59:18.326017Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.964495Z","title":"Rouge: a package for automatic evaluation of summaries,","venue":null,"work_id":"c05414de-b77e-4eca-a4d3-7556cb6a77e5","year":2004},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.330973Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:b3a041e5f393c0857e75c6f7dd0d403e1eba3d2d8f81736b8d072aeea9d4604c","observation_id":"9724063b-752b-41cc-92a5-2e8b9e3fa414","resolution":{"observed_at":"2026-08-12T11:59:18.968535Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.07503","last_updated":"2024-08-10T20:46:47Z","snapshot_observed_at":"2026-08-13T00:32:55.965962Z","submitted_at":"2024-04-11T06:34:17Z","title":"Best Practices and Lessons Learned on Synthetic Data","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.07503","snapshot_observed_at":"2026-08-12T11:59:18.334631Z","title":"Best practices and lessons learned on synthetic data for language models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.334631Z"},"links":{"cited_paper":"/paper/2404.07503","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:d351016afb503db97bbf37afccb4b832ba8fd23d573c471c986c3d9b623947ef","observation_id":"56a5924d-8edc-4c5f-8af5-dc72367b80be","resolution":{"observed_at":"2026-08-12T11:59:18.334631Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.13308","last_updated":"2023-09-23T08:46:11Z","snapshot_observed_at":"2026-08-13T10:07:09.184755Z","submitted_at":"2023-09-23T08:46:11Z","title":"Calibrating LLM-Based Evaluator","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.13308","snapshot_observed_at":"2026-08-12T11:59:18.339313Z","title":"Calibrating llm-based evaluator,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.339313Z"},"links":{"cited_paper":"/paper/2309.13308","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:89ee2d1530c0a5326e450974ed6095a08192855021f7dc740d364a044805e744","observation_id":"a57f28fd-7676-4364-9c5f-b975c931315e","resolution":{"observed_at":"2026-08-12T11:59:18.339313Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.952500Z","title":"Generating training data with language models: towards zero-shot language understanding,","venue":null,"work_id":"79b14e01-1eae-4969-8ad7-d92f35a32f57","year":2022},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.343429Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:7bdc773d53c7ef3b04cb7d59c03b3ea0f32f6ea16e4b19d1dcdf4e0258c148d6","observation_id":"ae9b3172-cb8b-4264-b00e-b1fc5795ef68","resolution":{"observed_at":"2026-08-12T11:59:18.956636Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.940463Z","title":"Cider: robust consensus-based image description evaluation,","venue":null,"work_id":"c64e4fe6-6856-48f5-ac51-0d2d3847df0c","year":2021},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.348047Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:dfa647fe84020384469b0ae82171663a0b8e7ccbc65765c9509b362655e91d26","observation_id":"eb717707-4edb-4c03-aeb5-23bf499ef0a4","resolution":{"observed_at":"2026-08-12T11:59:18.944650Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.927793Z","title":"A framework for evaluating and improving requirements specifications based on the developers and testers perspective,","venue":null,"work_id":"5193e3b9-38a2-40d6-bc3f-db546cdc30da","year":2021},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.352122Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:c197cf765a8a134348472e099906c536a630a7b80a15d6bd36b2154dcf135639","observation_id":"95a55474-bc6c-4ec1-81f6-73192c62e639","resolution":{"observed_at":"2026-08-12T11:59:18.932435Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.03479","last_updated":"2024-07-03T19:53:47Z","snapshot_observed_at":"2026-08-12T23:29:17.040194Z","submitted_at":"2024-07-03T19:53:47Z","title":"Human-Centered Design Recommendations for LLM-as-a-Judge","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.03479","snapshot_observed_at":"2026-08-12T11:59:18.356623Z","title":"Human-centered design recommendations for llm-as-a-judge,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.356623Z"},"links":{"cited_paper":"/paper/2407.03479","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:a3b778669195ebd13b525240f216a7ffee9271351b01f0ff742c880f5103c1b7","observation_id":"698c9c48-3d7c-4663-9190-028b2013d533","resolution":{"observed_at":"2026-08-12T11:59:18.356623Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.915368Z","title":"Bleu: a method for automatic evaluation of machine translation,","venue":null,"work_id":"83134da3-ae6d-4f7c-a78e-faae5eb88a75","year":2002},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.360737Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:ec8cfde74b6f039bbe44e6040648b45792f46fb87fecb839a84011a9722c24bc","observation_id":"d380be56-886c-43af-a80c-d04386398aaf","resolution":{"observed_at":"2026-08-12T11:59:18.920316Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2408.13296","last_updated":"2024-10-30T01:04:15Z","snapshot_observed_at":"2026-08-14T02:11:16.434887Z","submitted_at":"2024-08-23T14:48:02Z","title":"The Ultimate Guide to Fine-Tuning LLMs from Basics to Breakthroughs: An Exhaustive Review of Technologies, Research, Best Practices, Applied Research Challenges and Opportunities","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.13296","snapshot_observed_at":"2026-08-12T11:59:18.365781Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.365781Z"},"links":{"cited_paper":"/paper/2408.13296","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:3a931c40a00932bca4cc13810c0e8b1ce72403e411aa8ab6a2748257b65ff5c8","observation_id":"a5a90d46-9e76-4ecb-94ce-4bf898a63156","resolution":{"observed_at":"2026-08-12T11:59:18.365781Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.10076","last_updated":"2023-10-16T05:19:02Z","snapshot_observed_at":"2026-08-13T05:48:49.313792Z","submitted_at":"2023-10-16T05:19:02Z","title":"Verbosity Bias in Preference Labeling by Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.10076","snapshot_observed_at":"2026-08-12T11:59:18.369902Z","title":"Verbosity bias in preference labeling by large language models,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.369902Z"},"links":{"cited_paper":"/paper/2310.10076","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:64a3e48853f853f0f4a564aee42fbfb7f14b0ea51009336d95c1c93d1f63ef81","observation_id":"57920572-3461-49cf-b1b9-5a97faa05cee","resolution":{"observed_at":"2026-08-12T11:59:18.369902Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2004.04696","last_updated":"2020-05-21T16:53:47Z","snapshot_observed_at":"2026-08-10T19:14:01.245716Z","submitted_at":"2020-04-09T17:26:52Z","title":"BLEURT: Learning Robust Metrics for Text Generation","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2004.04696","snapshot_observed_at":"2026-08-12T11:59:18.374841Z","title":"Bleurt: learning robust metrics for text generation,","venue":null,"work_id":null,"year":2004},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.374841Z"},"links":{"cited_paper":"/paper/2004.04696","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:8c7b13e46dd4a7846763b72892cb8d6c4f53f206e5bde979962615cbc7fe5318","observation_id":"0f1b10a0-d706-48ae-a6fd-e0e483db6f0e","resolution":{"observed_at":"2026-08-12T11:59:18.374841Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.902491Z","title":"On automatic summarization of what and why information in source code changes,","venue":null,"work_id":"cd6cdfce-ad2c-4c13-bf26-310ae90dd5c5","year":2016},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.379788Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:f5fd6b0255d6c15fb4e05d4865044ebd54ab267674f64c20cf42a146275e974c","observation_id":"1cd8717b-f6b1-4089-aec4-d5b796dbcae0","resolution":{"observed_at":"2026-08-12T11:59:18.906996Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.890035Z","title":"On the evaluation of neural code summarization,","venue":null,"work_id":"047771d6-7d01-48c2-9e3b-a9ca60b338da","year":2022},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.383721Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:653f18a447cf3ca9a1e48ba384ff4df4c235c57048ece48f60e0d66b6edb473b","observation_id":"ff1d24d0-ab0f-4ea3-92d4-57d3d1841f88","resolution":{"observed_at":"2026-08-12T11:59:18.894287Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.878433Z","title":"RACE: Retrieval-augmented commit message generation,","venue":null,"work_id":"58e2f551-2991-4b33-a25a-a0a7f35e1954","year":2022},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.388673Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:a0979a3acb77be8a6bb32afafe3030d5137b91a3e1680f1909c609505f6c408e","observation_id":"f1f2be3f-b8ef-4e6c-9a72-9c183b1bb0f9","resolution":{"observed_at":"2026-08-12T11:59:18.882388Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.866510Z","title":"On the evaluation of commit message generation models: an experimental study,","venue":null,"work_id":"b9891dce-5e2e-4133-84ea-347ce1c49822","year":2021},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.392463Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:88e8137c36aada483266d2e9bbea1c0fedfe8057f854b451584ff059d22d2878","observation_id":"9de57533-72ab-447b-9c2e-9a16d1936911","resolution":{"observed_at":"2026-08-12T11:59:18.870426Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.852877Z","title":"Alpaca: a strong, replicable instruction-following model,","venue":null,"work_id":"d04c6ba1-2f21-4a22-aecf-e9fdbde89075","year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.397190Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:e2a74baaba3fde02d33fb519348fc101e5dd1ea2eada6ba812862d81f70988f6","observation_id":"68bbccb3-869f-49bb-aa65-0f68b69ef797","resolution":{"observed_at":"2026-08-12T11:59:18.858724Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.12624","last_updated":"2025-08-18T00:07:23Z","snapshot_observed_at":"2026-08-12T23:40:03.945084Z","submitted_at":"2024-06-18T13:49:54Z","title":"Judging the Judges: Evaluating Alignment and Vulnerabilities in LLMs-as-Judges","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.12624","snapshot_observed_at":"2026-08-12T11:59:18.400788Z","title":"Judging the judges: evaluating alignment and vulnerabilities in llms-as-judges,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.400788Z"},"links":{"cited_paper":"/paper/2406.12624","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:97d225d32629d3b44a5f50e87f2c338a270e713c025aaa3c8c80a5c88224db66","observation_id":"196eee81-2147-4992-9d96-4f60defcddda","resolution":{"observed_at":"2026-08-12T11:59:18.400788Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.840041Z","title":"Synthetic data, real errors: how (not) to publish and use synthetic data,","venue":null,"work_id":"168b336f-8c65-43a5-9de0-f19821c4f64a","year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.405510Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:bde123ae924c43eacfd45278366a15ef58a5eb7a2e2fb34cc0b7b96129e46fdc","observation_id":"d64aa7a2-c591-4f8b-8884-5cb6100a9cc3","resolution":{"observed_at":"2026-08-12T11:59:18.844266Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.18796","last_updated":"2024-05-01T15:37:11Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-04-29T15:33:23Z","title":"Replacing Judges with Juries: Evaluating LLM Generations with a Panel of Diverse Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.18796","snapshot_observed_at":"2026-08-12T11:59:18.409126Z","title":"Replacing judges with juries: evaluating llm generations with a panel of diverse models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":64,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.409126Z"},"links":{"cited_paper":"/paper/2404.18796","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:0ab3ff1ef5f12e86e60741ad8b59bc6a9e2480b269d55df806de7b3744d8acc9","observation_id":"a7d2c3be-6852-4eea-b3be-9e62162e1d76","resolution":{"observed_at":"2026-08-12T11:59:18.409126Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.13006","last_updated":"2025-03-30T17:59:47Z","snapshot_observed_at":"2026-08-14T05:07:24.912137Z","submitted_at":"2024-08-23T11:49:01Z","title":"Systematic Evaluation of LLM-as-a-Judge in LLM Alignment Tasks: Explainable Metrics and Diverse Prompt Templates","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.13006","snapshot_observed_at":"2026-08-12T11:59:18.413886Z","title":"Systematic evaluation of llm-as-a-judge in llm alignment tasks: explainable metrics and diverse prompt templates,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":65,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.413886Z"},"links":{"cited_paper":"/paper/2408.13006","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:d3d3510595f7fc7ccf2cb67a438e04c94611cb42aff60abb2f0733c65c829fdd","observation_id":"02d0b65c-f5f1-4087-b51f-22bad0f786a2","resolution":{"observed_at":"2026-08-12T11:59:18.413886Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.04014","last_updated":"2024-07-04T15:45:15Z","snapshot_observed_at":"2026-08-13T10:16:42.630726Z","submitted_at":"2024-07-04T15:45:15Z","title":"Offline Energy-Optimal LLM Serving: Workload-Based Energy Models for LLM Inference on Heterogeneous Systems","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.04014","snapshot_observed_at":"2026-08-12T11:59:18.417892Z","title":"Offline energy-optimal llm serv- ing: Workload-based energy models for llm inference on heterogeneous systems,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":66,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.417892Z"},"links":{"cited_paper":"/paper/2407.04014","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:51bc2a42d0fb250afe5b9dfde947a90c406bdf624e8137e4795a0ae04d364094","observation_id":"5746f32f-31dc-40ec-b425-a3aa15be497f","resolution":{"observed_at":"2026-08-12T11:59:18.417892Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.827971Z","title":"Fake it till you make it: face analysis in the wild using synthetic data alone,","venue":null,"work_id":"14449594-5829-489c-ab70-c3cc68e7297b","year":2021},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":67,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.422862Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:e8704179367e5a96a5e473a7e150aacfd7fa74faace87b4e39e34e1afbb51eec","observation_id":"99eedc02-be11-4d1c-bb22-864eaef6e7e4","resolution":{"observed_at":"2026-08-12T11:59:18.831987Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.816236Z","title":"Commit message generation via chatgpt: how far are we?","venue":null,"work_id":"31fa2c1e-6af4-4804-97d1-da4127d5e13d","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":68,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.427237Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:808ce5d87f3563e4bf6eed05ab0c6c9082b6f4f14ba5501358a78f107b07b69b","observation_id":"219aebda-04d4-455d-b19a-6145c4cbea0a","resolution":{"observed_at":"2026-08-12T11:59:18.820449Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.804402Z","title":"Automatic commit message generation: a critical review and directions for future work,","venue":null,"work_id":"c509f08b-9099-4fdc-97b6-67c5499530a6","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":69,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.431034Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:365d4fd5cf2a68a12fd4c1531eaa63bd60f94962fb284bce38a0afc1277f2b82","observation_id":"f890a40e-3532-4f3d-b0f0-0e99dbe9cb2c","resolution":{"observed_at":"2026-08-12T11:59:18.808693Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.789857Z","title":"Judging llm-as-a-judge with mt-bench and chatbot arena,","venue":null,"work_id":"4a985510-5646-4281-870e-1538dfbe5f9e","year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":70,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.434546Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:cdaca715f71b554d31664d3162b0a87f0536ab6f6383a0a3405371e3a3149594","observation_id":"b3e532f0-e1c9-45c0-8e81-6851a8f47262","resolution":{"observed_at":"2026-08-12T11:59:18.795757Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","latest_version":1,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-13T15:30:04.590669Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems"},"reference_resolution":{"displayed":70,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":28,"verified_exact":0,"verified_fuzzy":42},"total_outbound_references":70},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"thesis":"As of 14 August 2026, this Paper Citation Record lists 70 of 70 outbound references and 2 inbound Pith citation observations for arXiv:2411.17793."}