{"as_of":"2026-08-18T16:29:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:8c15faffbf7b08bc30177dfe2e5f38945d6fb681ed95f0ff656836b9db755916","coverage":[{"denominator":23,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":23,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T16:45:16.249314Z","state":"measured"},{"denominator":23,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":23,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-18T06:34:40.430872+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2411.13212/citation-record","integrity":"/paper/2411.13212/integrity","json":"/paper/2411.13212/citation-record.json","paper":"/paper/2411.13212"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.172964Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.172964Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:cb90750682b8177451a0d2dacf1183bd388b151ee49d7aa25c6a1926c147f65c","observation_id":"719a5616-f964-4136-bae7-31952d4b207c","resolution":{"observed_at":"2026-08-12T16:45:16.172964Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.181554Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.181554Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:4482cd8ceac534481727af1c274d406ee405dafff8dcecf3a847c76b7a832316","observation_id":"23c5bd0d-aa97-4efe-8f17-2fa2e8b26ebb","resolution":{"observed_at":"2026-08-12T16:45:16.181554Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.188659Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.188659Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:18f851a8435c77f9f33754f0f6b12f4950d1b209f61d0aa6b8b1a458f9058f5c","observation_id":"11a01bd5-e960-49ab-af5e-b27b3506cbf1","resolution":{"observed_at":"2026-08-12T16:45:16.188659Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.951754Z","title":null,"venue":null,"work_id":"3237ac24-e2e1-4648-83d8-25c6e2bc5da5","year":1948},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.192143Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:89b9fbadbf98fab4ae03b69dc93c5ac59412c9bc462f4eb36f22314e768644a6","observation_id":"ba6f85e9-2379-493d-be2a-9cf6a11b8fc5","resolution":{"observed_at":"2026-08-12T16:45:16.954953Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.195698Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.195698Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:47b08c17af61ac366ca18f3ba4d3730c6de5ff41b2e153d8fd99e1f15421b9a0","observation_id":"a319de42-8823-401c-ae81-93c3e7d9b326","resolution":{"observed_at":"2026-08-12T16:45:16.195698Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.199067Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.199067Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:ca807885c13f16a20dc615d477165a57ca2a488e5d372886ff304aa71cc7b6f1","observation_id":"2877a261-d1ea-46e7-baae-f3e2868fecbd","resolution":{"observed_at":"2026-08-12T16:45:16.199067Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1177/01","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.283134Z","title":null,"venue":null,"work_id":"45730cd3-af8a-44c6-8a29-2ebb47aa959f","year":2023},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.202283Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:24b8f71e51a7203d1c5ac2f23781a0497dc8542bd48a483bfd9ec22dd755e866","observation_id":"86fd2ccb-ce24-4e1e-8005-5574ed2bc5c5","resolution":{"observed_at":"2026-08-12T16:45:16.286367Z","resolver_source":"doi_truncated","status":"malformed_identifier"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.942375Z","title":null,"venue":null,"work_id":"fccd8015-0bf2-498c-ba85-1b80ebdb59cd","year":2025},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.205442Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:c4d222fa014c8b8b2bdd3b1d1f0189381b5832e659382d4c9856b1fa1d705fec","observation_id":"47810618-584b-478e-b3ab-0dc86d1b7c13","resolution":{"observed_at":"2026-08-12T16:45:16.945776Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.208443Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.208443Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:4ef1db344bdcdbfacbdcfd3ecaf9e718dbd0c37c71c0f5012b63b966a74a8f5c","observation_id":"63177dff-b5d9-4d3e-957b-3c3e97414cb9","resolution":{"observed_at":"2026-08-12T16:45:16.208443Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.211432Z","title":"Losada, and Álvaro Barreiro","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.211432Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:29f7e9d7cba41cc7a1a466a534394605b23e0658e17b3dd072e16a538cddc056","observation_id":"6cb2ca0e-e5b9-4f25-8638-1a7657acc8a2","resolution":{"observed_at":"2026-08-12T16:45:16.211432Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.214392Z","title":"Rahmani, Nick Craswell, Emine Yilmaz, Bhaskar Mitra, and Daniel Campos","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.214392Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:ef0865fa6eae650a54545492be8ebd69f1bf7cbaa1e49c147bcf11ea6dfb1e4c","observation_id":"0f406067-1086-452a-9b58-e535f94cef88","resolution":{"observed_at":"2026-08-12T16:45:16.214392Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.16312","last_updated":"2025-01-25T16:30:57Z","snapshot_observed_at":"2026-08-18T05:10:38.593764Z","submitted_at":"2024-08-29T07:20:56Z","title":"SynDL: A Large-Scale Synthetic Test Collection for Passage Retrieval","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.16312","snapshot_observed_at":"2026-08-12T16:45:16.217587Z","title":"Rahmani, Xi Wang, Emine Yilmaz, Nick Craswell, Bhaskar Mitra, and Paul Thomas","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.217587Z"},"links":{"cited_paper":"/paper/2408.16312","citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:1751b0eaba9f1c3e76d55bc5f6e3a148b4092ac9c7bee088d96abfb5fd93bdb9","observation_id":"1b0a63bc-0732-433b-b94f-57f2d74f53fb","resolution":{"observed_at":"2026-08-12T16:45:16.217587Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.220944Z","title":null,"venue":null,"work_id":null,"year":2005},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.220944Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:dba09ac101f338b20e00dafd94bdca25a90deb64c3af701f76d6a118f6e9b8f4","observation_id":"a75077f7-f433-44f9-95b8-7acf7330784d","resolution":{"observed_at":"2026-08-12T16:45:16.220944Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.224025Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.224025Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:6c14dda783abbd21d794f38921ee9c7aae26533b5df35609eb48a2915ec12806","observation_id":"2d70b950-2328-4124-8d5b-f7841ce428da","resolution":{"observed_at":"2026-08-12T16:45:16.224025Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.04727","last_updated":"2024-05-08T00:32:19Z","snapshot_observed_at":"2026-08-16T18:49:16.197573Z","submitted_at":"2024-05-08T00:32:19Z","title":"LLMs Can Patch Up Missing Relevance Judgments in Evaluation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.04727","snapshot_observed_at":"2026-08-12T16:45:16.227040Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.227040Z"},"links":{"cited_paper":"/paper/2405.04727","citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:7b609e6885b7d0f0fa239a57ccfd0b213ae0367984f5146b4aedd8e67be2a216","observation_id":"2d0f5d2f-ac9c-4096-81c6-035bc5fa079b","resolution":{"observed_at":"2026-08-12T16:45:16.227040Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.08275","last_updated":"2024-11-13T01:12:35Z","snapshot_observed_at":"2026-08-13T00:35:28.098798Z","submitted_at":"2024-11-13T01:12:35Z","title":"A Large-Scale Study of Relevance Assessments with Large Language Models: An Initial Look","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.08275","snapshot_observed_at":"2026-08-12T16:45:16.230608Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.230608Z"},"links":{"cited_paper":"/paper/2411.08275","citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:ee2d39331fd4fbc994f76a4b79a5c657efd704c6fbcd5db00fee613285209eca","observation_id":"d9a39eb1-883e-4ef9-8e2f-d267468b9c6c","resolution":{"observed_at":"2026-08-12T16:45:16.230608Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.06519","last_updated":"2024-06-10T17:58:29Z","snapshot_observed_at":"2026-08-16T18:21:09.288186Z","submitted_at":"2024-06-10T17:58:29Z","title":"UMBRELA: UMbrela is the (Open-Source Reproduction of the) Bing RELevance Assessor","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.06519","snapshot_observed_at":"2026-08-12T16:45:16.233934Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.233934Z"},"links":{"cited_paper":"/paper/2406.06519","citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:c5e17e40a39cf7557f8f67ca19e9ef67ce9fdc64f1be64681e58bbe94b93cf44","observation_id":"a28b4541-130e-4fb8-aad0-e16e78c42fa9","resolution":{"observed_at":"2026-08-12T16:45:16.233934Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.237208Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.237208Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:2818bde38387bc33a30c169bd5892755251058b7a0c7408cd545bf5aca89a50e","observation_id":"3173b6b2-a376-491c-b094-64c0585e645a","resolution":{"observed_at":"2026-08-12T16:45:16.237208Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1145/3269","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.272457Z","title":"Voorhees","venue":null,"work_id":"754a2f47-cc57-48e5-94f8-59267143c43f","year":2018},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.240158Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:66a75083afd7880e0e733017c689f93b72d885257e73590e58ef2a9c03ac507f","observation_id":"fb359f50-d6b9-4044-a36c-19f311244491","resolution":{"observed_at":"2026-08-12T16:45:16.276630Z","resolver_source":"doi_truncated","status":"malformed_identifier"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.243268Z","title":null,"venue":null,"work_id":null,"year":2010},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.243268Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:7fbb510ba48c44568c5b9344e59d60447cdca98596e245509197ab3dd620064f","observation_id":"03e2af04-5850-4189-8f0e-3026fdc512b7","resolution":{"observed_at":"2026-08-12T16:45:16.243268Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.246309Z","title":null,"venue":null,"work_id":null,"year":2008},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.246309Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:21b8ecbe04eb156a932ee4afd9b17ea1792b7cee72d77b344472c67a0128b7e9","observation_id":"239b7110-afd6-451b-a258-12c7eaa8d807","resolution":{"observed_at":"2026-08-12T16:45:16.246309Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:45:16.249314Z","title":"Aslam, and Stephen Robertson","venue":null,"work_id":null,"year":2008},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.249314Z"},"links":{"citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:c3d2c6e87ffc029c4004f628a01067d25deb2adfa50e2aba1cd228a41231d38d","observation_id":"183aa483-9e76-4d1a-9cda-f107feacfb56","resolution":{"observed_at":"2026-08-12T16:45:16.249314Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.05600","last_updated":"2024-05-09T07:39:19Z","snapshot_observed_at":"2026-08-16T13:54:07.456304Z","submitted_at":"2024-05-09T07:39:19Z","title":"Can We Use Large Language Models to Fill Relevance Judgment Holes?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.05600","snapshot_observed_at":"2026-08-12T16:45:16.177722Z","title":"https://arxiv.org/abs/2405.05600 arXiv: 2405.05600 [cs.IR] (cited on p","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation","version":3},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-12T16:45:16.177722Z"},"links":{"cited_paper":"/paper/2405.05600","citing_paper":"/paper/2411.13212"},"observation_digest":"sha256:b5324d132edd100c63fb7ee192da0d38646ba9181a1213148e727cf86bef0001","observation_id":"33d3d959-0248-4a3c-9cd5-4354eb5fa3d6","resolution":{"observed_at":"2026-08-12T16:45:16.177722Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2411.13212","last_updated":"2025-04-15T09:11:18Z","latest_version":3,"primary_category":"cs.IR","snapshot_observed_at":"2026-08-16T06:44:47.881324Z","submitted_at":"2024-11-20T11:19:35Z","title":"Limitations of Automatic Relevance Assessments with Large Language Models for Fair and Reliable Retrieval Evaluation"},"reference_resolution":{"displayed":23,"state_counts":{"malformed_identifier":2,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":21,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":23},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"thesis":"As of 18 August 2026, this Paper Citation Record lists 23 of 23 outbound references and 0 inbound Pith citation observations for arXiv:2411.13212."}