{"as_of":"2026-08-10T08:27:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:82c3b5d4d084f92952fa1d70d8553d49a79fec4db985d507fe09b0ffee227e34","coverage":[{"denominator":38,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":38,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-06T18:00:21.298350Z","state":"measured"},{"denominator":43,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":43,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-10T06:31:04.303077+00:00","state":"measured"},{"denominator":5,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":5,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-03T03:31:57.025464Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-03T22:49:00.988765Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2507.09488","snapshot_observed_at":"2026-08-03T03:31:57.025464Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2602.07840","last_updated":"2026-06-10T04:03:01Z","snapshot_observed_at":"2026-08-09T01:00:07.289074Z","submitted_at":"2026-02-08T06:42:50Z","title":"SAGE: Scalable AI Governance & Evaluation","version":4},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-03T03:31:57.025464Z"},"links":{"cited_paper":"/paper/2507.09488","citing_paper":"/paper/2602.07840"},"observation_digest":"sha256:933ec644a2d6a03acbf4b427186055007c68dc967704212dfc89f7b92b5cef0d","observation_id":"cd4fbf59-8fa1-45e0-8e0f-cdaf24cacc6a","resolution":{"observed_at":"2026-08-03T03:31:57.025464Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"cited_work":{"arxiv_id":"2507.09488","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2507.09488","snapshot_observed_at":"2026-07-03T22:49:00.988765Z","title":"arXiv:2507.09488 [cs.IR] https://arxiv.org/ abs/2507.09488 Di Jin, Eileen Pan, Nassim Oufattole, Wei-Hung Weng, Hanyi Fang, and Peter Szolovits","venue":null,"work_id":"e259fc31-0740-47c1-929d-13ffafc626f3","year":2025},"citing_paper":{"arxiv_id":"2606.29467","last_updated":"2026-07-06T10:42:45Z","snapshot_observed_at":"2026-07-12T10:57:57.150115Z","submitted_at":"2026-06-28T15:51:53Z","title":"mamabench and mamaretrieval: Benchmarks for Evaluating Medical Retrieval-Augmented Generation in Maternal, Neonatal, and Reproductive Health","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-06-30T07:39:08.660337Z"},"links":{"cited_paper":"/paper/2507.09488","citing_paper":"/paper/2606.29467"},"observation_digest":"sha256:0384835434e6f01ee71640fd1658d50b01bc04a26b3e9aa7f1f983a376806ecd","observation_id":"ec6795ed-98e8-4102-9ec4-b83d165037c6","resolution":{"observed_at":"2026-06-30T07:44:21.974095Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"cited_work":{"arxiv_id":"2507.09488","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2507.09488","snapshot_observed_at":"2026-07-03T22:49:00.988765Z","title":"arXiv:2507.09488 [cs.IR] https://arxiv.org/ abs/2507.09488 Di Jin, Eileen Pan, Nassim Oufattole, Wei-Hung Weng, Hanyi Fang, and Peter Szolovits","venue":null,"work_id":"e259fc31-0740-47c1-929d-13ffafc626f3","year":2025},"citing_paper":{"arxiv_id":"2606.29467","last_updated":"2026-07-06T10:42:45Z","snapshot_observed_at":"2026-07-12T10:57:57.150115Z","submitted_at":"2026-06-28T15:51:53Z","title":"mamabench and mamaretrieval: Benchmarks for Evaluating Medical Retrieval-Augmented Generation in Maternal, Neonatal, and Reproductive Health","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-07-03T22:42:40.207711Z"},"links":{"cited_paper":"/paper/2507.09488","citing_paper":"/paper/2606.29467"},"observation_digest":"sha256:7cd827b8fe3bf2ede1b3f658cae4b077c2ca7d2d49b8db3c37593920bfc0c657","observation_id":"0a0bfecf-24fa-4839-b528-ead159cc4826","resolution":{"observed_at":"2026-07-03T22:49:00.991852Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2507.09488","snapshot_observed_at":"2026-07-12T10:57:57.406511Z","title":"arXiv:2507.09488 [cs.IR] https://arxiv.org/ abs/2507.09488 Di Jin, Eileen Pan, Nassim Oufattole, Wei-Hung Weng, Hanyi Fang, and Peter Szolovits","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2606.29467","last_updated":"2026-07-06T10:42:45Z","snapshot_observed_at":"2026-07-12T10:57:57.150115Z","submitted_at":"2026-06-28T15:51:53Z","title":"mamabench and mamaretrieval: Benchmarks for Evaluating Medical Retrieval-Augmented Generation in Maternal, Neonatal, and Reproductive Health","version":3},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-07-12T10:57:57.406511Z"},"links":{"cited_paper":"/paper/2507.09488","citing_paper":"/paper/2606.29467"},"observation_digest":"sha256:2038fd204a2d58d65f14ec4dfc240142da4e445688f6a4cf1ab4ffb35c67626b","observation_id":"5c3361bd-f690-4cfc-9754-a09c8b02a848","resolution":{"observed_at":"2026-07-12T10:57:57.406511Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"cited_work":{"arxiv_id":"2507.09488","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2507.09488","snapshot_observed_at":"2026-07-03T22:49:00.988765Z","title":"arXiv:2507.09488 [cs.IR] https://arxiv.org/ abs/2507.09488 Di Jin, Eileen Pan, Nassim Oufattole, Wei-Hung Weng, Hanyi Fang, and Peter Szolovits","venue":null,"work_id":"e259fc31-0740-47c1-929d-13ffafc626f3","year":2025},"citing_paper":{"arxiv_id":"2606.30840","last_updated":"2026-06-29T19:16:07Z","snapshot_observed_at":"2026-08-06T13:28:24.425801Z","submitted_at":"2026-06-29T19:16:07Z","title":"Contrastive Reflection for Iterative Prompt Optimization","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-07-01T02:02:22.841488Z"},"links":{"cited_paper":"/paper/2507.09488","citing_paper":"/paper/2606.30840"},"observation_digest":"sha256:8bb1b2f477f9def9b7d92be915b82283c73ac9b8e5d0ccf0389600bc8b911ee6","observation_id":"c43ef256-c6fb-4288-9f91-6d797feb36f6","resolution":{"observed_at":"2026-07-01T12:25:44.293921Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2507.09488/citation-record","integrity":"/paper/2507.09488/integrity","json":"/paper/2507.09488/citation-record.json","paper":"/paper/2507.09488"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2405.05600","last_updated":"2024-05-09T07:39:19Z","snapshot_observed_at":"2026-08-09T00:59:20.809933Z","submitted_at":"2024-05-09T07:39:19Z","title":"Can We Use Large Language Models to Fill Relevance Judgment Holes?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.05600","snapshot_observed_at":"2026-08-06T18:00:21.149269Z","title":"Can we use large language models to fill relevance judgment holes? arXiv preprint arXiv:2405.05600, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.149269Z"},"links":{"cited_paper":"/paper/2405.05600","citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:85206c39ca3d16ed24bb0beab9e4614dfcb6b4f2c08d5888330cc0922db0572b","observation_id":"4e55de38-3c98-46fa-bc43-2ca872946aeb","resolution":{"observed_at":"2026-08-06T18:00:21.149269Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.713532Z","title":"Llms can be fooled into labelling a document as relevant: best café near me; this paper is perfectly relevant","venue":null,"work_id":"67271a80-a0de-4c1a-b737-c910a73938c9","year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.153555Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:5187d65abfbf6ead717fb14996d7403ba657acd46f5bafe7e1365e79fb7fd44d","observation_id":"48a7e36b-3a97-4b99-a9f7-ab394badd637","resolution":{"observed_at":"2026-08-06T18:00:21.716947Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.703353Z","title":null,"venue":null,"work_id":"0b864ca8-0b6d-4e9c-bb32-a45f680ebf65","year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.157366Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:92b319908f879e77190d462d51582a33f5e343f5ba7fde07b468ef01665efde3","observation_id":"3295e0ee-6269-446b-9f02-5754c25466e7","resolution":{"observed_at":"2026-08-06T18:00:21.706807Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.693686Z","title":"Barry and Linda Schamber","venue":null,"work_id":"3eb2bab7-36b8-4e74-9b63-27fb3be5e992","year":1998},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.161831Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:de22f80e40ad1586a93da577b1f9b0c38bfece07362b6555d994f28dcd2d927e","observation_id":"f386242e-79cf-439c-bbb8-15f8afa5f40d","resolution":{"observed_at":"2026-08-06T18:00:21.696511Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.683588Z","title":"The concept of relevance in IR","venue":null,"work_id":"3dad04d2-3b5b-4da9-87e3-5e723c921db5","year":2003},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.166395Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:808f59f4a55cf08c261806d72d5eebd4027703d85bd6dd76d46a3a5f92879972","observation_id":"0098a147-bcab-4f39-9460-959af3e8bba2","resolution":{"observed_at":"2026-08-06T18:00:21.687135Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.673400Z","title":"Beyond topicality: A two stage view of relevance and the retrieval process","venue":null,"work_id":"a45ef8bb-9d42-4469-84a9-3b7cde276cba","year":1982},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.171127Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:c0ad8a1c9a9c059967f90a6a5ddaf284bee5b0b4d3be7b44b7c9e22a96ae275f","observation_id":"d019da2b-9c61-447b-9461-634af7fb4d10","resolution":{"observed_at":"2026-08-06T18:00:21.676959Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.662899Z","title":null,"venue":null,"work_id":"e18bfce4-6ef6-4bbd-858b-87dae5a64c27","year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.175553Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:97e2bc950e1f35d95410a2db63ef1ff1742f2e677a98257adbc579963fefa839","observation_id":"8bc23af9-7059-41b9-b8c1-052ce236e32b","resolution":{"observed_at":"2026-08-06T18:00:21.666517Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.651739Z","title":"Dimensions of Relevance","venue":null,"work_id":"f82a35a5-cfba-4043-9abd-b1986333990b","year":2000},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.179318Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:0e3de0a2aee56e52ed3036f92c32688625e6fbacef680fe3319a6d4d79e82a13","observation_id":"6be0c392-5cdf-4471-af07-11c8d05cf50c","resolution":{"observed_at":"2026-08-06T18:00:21.655170Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.641880Z","title":"Overview of the TREC 2020 deep learning track","venue":null,"work_id":"e3a414ff-eaef-4473-a988-300941fadbf7","year":2020},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.182993Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:0c4966ac71090790b29a3b4067aa1518dd2feb9b7e2d0ee610db2be6a218b20e","observation_id":"92fa95d9-72ee-450b-be99-12f718c485c7","resolution":{"observed_at":"2026-08-06T18:00:21.645451Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.630415Z","title":"Voorhees","venue":null,"work_id":"f094ddd0-e461-47f4-bc3c-bbb9509aefd6","year":2019},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.186902Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:4815722a00496a7dc258babc59d9ff315be8af835e2860e10283694c06ec1517","observation_id":"9f9e401e-4b30-46a6-b60e-3e20b6f2f60f","resolution":{"observed_at":"2026-08-06T18:00:21.634551Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.619250Z","title":"A workbench for autograding retrieve/generate systems","venue":null,"work_id":"a39643ec-b4c7-4ea5-9fc2-4a763fcb1b16","year":1963},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.190464Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:39eb421614506512aa0cbea6e2964249e594f3a32b5024fe1863f3fe099ff7a6","observation_id":"a75ccc18-3ed1-44e7-af8d-6d4030693cfd","resolution":{"observed_at":"2026-08-06T18:00:21.623238Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.608125Z","title":"Principles and guidelines for the use of llm judges","venue":null,"work_id":"3938c668-1b08-4de2-92d0-98a5ec8c1e1a","year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.194551Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:776f5c6fedbb77bcb389001d3ba517c54d8f875780d2f57c093d80378bb9516d","observation_id":"3f42af73-244e-48e9-bae0-16c21eb56037","resolution":{"observed_at":"2026-08-06T18:00:21.612033Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.198187Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.198187Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:577ba62d580cdb54eee3f8c0518aa29ee1a85f3116a146e1d46bab5e21838e30","observation_id":"143437d1-a5a2-4b60-ae78-a195df2a83d9","resolution":{"observed_at":"2026-08-06T18:00:21.198187Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.590274Z","title":"EXAM++: LLM-based Answerability Metrics for IR Evaluation","venue":null,"work_id":"6f009ffe-4c27-45d8-94c0-2dddaf110659","year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.201897Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:b8dc0607d8c14faec31e84ee8cf3abdd91718564d5e0559ab9560b39879fc570","observation_id":"42120bb9-7e03-43ae-a938-60ff3c87f7ac","resolution":{"observed_at":"2026-08-06T18:00:21.593988Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.580705Z","title":"Pencils Down! Automatic Rubric-based Eval- uation of Retrieve/Generate Systems","venue":null,"work_id":"63dd1b58-6f05-455e-af16-93d3f446f766","year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.205458Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:ac6a354076889763c8f0843ba3f8b4e337fdb3ca3467a1cde2496e17ebe95896","observation_id":"fbfa5b5a-aef5-45dd-aaa5-a076b269dd02","resolution":{"observed_at":"2026-08-06T18:00:21.583974Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.569794Z","title":null,"venue":null,"work_id":"4e14fa93-30f8-4848-bd79-f5bd17a67535","year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.209170Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:17c1df2f16fd0df1eb1788861a575db11d630dfb80e5717e626e827e3ca97994","observation_id":"5f6a52a7-0155-4d6e-965c-b703378c7df0","resolution":{"observed_at":"2026-08-06T18:00:21.574174Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.560594Z","title":"Bruce Croft","venue":null,"work_id":"376aa7b7-d1da-466e-954c-08d451339285","year":2020},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.213347Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:9d9de35256e7d15c684ff6fd11addfe495afae191d89092db60308ffabdb262f","observation_id":"3bb5820a-542c-438f-9689-0f6013afbe0f","resolution":{"observed_at":"2026-08-06T18:00:21.563843Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.551264Z","title":"One-shot labeling for automatic relevance estimation","venue":null,"work_id":"45d666da-1558-4322-a250-df63316d8d2d","year":2023},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.217661Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:bc6e1c51ec3fd18de9a89e65da8bfb302b8cfeca82d2d95e4308a708dd53c064","observation_id":"6bdb910c-a3f0-482d-83e5-1f5a1024a4ed","resolution":{"observed_at":"2026-08-06T18:00:21.554495Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.541471Z","title":"Crisisfacts: building and evaluating crisis timelines","venue":null,"work_id":"77a9bd76-502f-477f-8a3b-35a3da0a477e","year":2023},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.221586Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:ca73c69ad7c725b186cdfd53e1df9fc98ab25262525c1ecd5a07a404a9b0355d","observation_id":"5432c96d-3fdb-4476-ab94-ec26a4e5c93e","resolution":{"observed_at":"2026-08-06T18:00:21.544892Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.530422Z","title":"Query performance prediction using relevance judgments generated by large language models","venue":null,"work_id":"ac79b9e5-0c79-4d47-910f-a0b2b749ec34","year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.225750Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:e9eff5b9d1387b0f9df88fdda47d8f03311b205c855e3c0e13a26b1d59a37654","observation_id":"616b0a1e-3378-443d-8267-79795fb19790","resolution":{"observed_at":"2026-08-06T18:00:21.535121Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.520341Z","title":"Golbus, and Javed A","venue":null,"work_id":"1e6e4a3e-8dc9-46b4-af59-db5421a89152","year":2012},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.230127Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:cbdcad65631551d5ccbcec41baacae4207a9ecee6b511d3e638dd67d09cb3c66","observation_id":"a14ee621-5997-4b67-afc8-2df1765166e3","resolution":{"observed_at":"2026-08-06T18:00:21.523900Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.510406Z","title":"Initial nugget evaluation results for the trec 2024 rag track with the autonuggetizer framework, 2024","venue":null,"work_id":"3348e362-4d71-423b-a7fe-ec6663db64cf","year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.234722Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:5ed50b6ca80e7346df84051854850e51433b64cefa0b07660a62e1be9e1b6e84","observation_id":"107a5f42-2112-4f1d-95ea-028290d05d1d","resolution":{"observed_at":"2026-08-06T18:00:21.514146Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.499872Z","title":"Rahmani, Varsha Ramineni, Nick Craswell, Bhaskar Mitra, and Emine Yilmaz","venue":null,"work_id":"b1104d96-3c55-4e72-ab77-8c3701ea346c","year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.239059Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:5f540dd8ac76a0d24d963ba55bb09ae3d6e07a3ddc13008620708e4009f5ebc4","observation_id":"6438008e-e4a7-4f81-92f4-d07e91ad4e6a","resolution":{"observed_at":"2026-08-06T18:00:21.503471Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.488605Z","title":"Rahmani, Clemencia Siro, Mohammad Aliannejadi, Nick Craswell, Charles L","venue":null,"work_id":"21e0635c-ac0b-442b-8378-4d0317425e5d","year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.242928Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:4ad7a3be0e8f523c04a9be87769dbfe5f88a0d9e560ddb2a051ca93d4a237cd9","observation_id":"29250cdc-d00d-4f55-9b9f-ddea7d2f741e","resolution":{"observed_at":"2026-08-06T18:00:21.492219Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.477555Z","title":"Rahmani, Emine Yilmaz, Nick Craswell, and Bhaskar Mitra","venue":null,"work_id":"513dc8db-03a4-4b84-9eec-24e87ba400f2","year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.246582Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:481c19250686ea32ce9f89df37907120ce19c6c791d935714498170f665f7134","observation_id":"ef6bbbdd-2315-4e54-821a-94cccf3ce142","resolution":{"observed_at":"2026-08-06T18:00:21.481141Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.466213Z","title":"Rahmani, Emine Yilmaz, Nick Craswell, Bhaskar Mitra, Paul Thomas, Charles L","venue":null,"work_id":"8ca06376-1dd4-4ca5-b9ea-e4e19ac92111","year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.250427Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:10e1f6e8ac852d604299b0abb64a0fd0331be2c00336cffe5395a255b28e83e6","observation_id":"cce1a809-3c56-4288-8703-0fb9976eb3cf","resolution":{"observed_at":"2026-08-06T18:00:21.470027Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.454817Z","title":"Swan: A generic framework for auditing textual conversational systems, 2023","venue":null,"work_id":"aebf078f-f35d-4683-a38f-5b7c7ab066e6","year":2023},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.255514Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:3ac8ff31b8e5a77518f72b74ef858791dca3db244ea8749536d8fd1f61cefa97","observation_id":"538a8fd2-61b5-4b21-a69f-daef79b94928","resolution":{"observed_at":"2026-08-06T18:00:21.458551Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.442475Z","title":"Exam: How to evaluate retrieve-and-generate systems for users who do not (yet) know what they want","venue":null,"work_id":"74d55b11-7e9f-4857-b646-cdb73af59040","year":2021},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.259126Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:eda30e013c02519c8a9b27af9b77db49a1ee00f27d1dc0394786b5ae9b10b26b","observation_id":"3b85eb03-9b97-4519-9550-4618a42a0a70","resolution":{"observed_at":"2026-08-06T18:00:21.446632Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.431286Z","title":"Relevance: A review of the literature and a framework for thinking on the notion in information science","venue":null,"work_id":"5af886ba-12dc-4181-b653-3d0ac903a456","year":1915},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.263223Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:b32c5a0c9d2a184afd012592c85bc8a62a30255bfc425c32e0864b7c447a6f89","observation_id":"6299d63e-10d7-481f-850f-34288542503b","resolution":{"observed_at":"2026-08-06T18:00:21.435613Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.267029Z","title":"Don’t use LLMs to make relevance judgments.Information Retrieval Research Journal, 1(1):10.54195/irrj.19625, March 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.267029Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:6c47d8237755a3592a3deb40d1d2fc1850299044bdd6760c0a8b1c4f1369f7b4","observation_id":"e9881564-ccd6-46d2-baf2-2f35c0fc55fd","resolution":{"observed_at":"2026-08-06T18:00:21.267029Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.420951Z","title":"Is ChatGPT good at search? investigating large language models as re-ranking agents","venue":null,"work_id":"b44f8115-6ac0-4cec-9243-04f331b5c923","year":2023},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.271399Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:3d8ca2b64d5319d5f0d2083da8d297a904c267f91dbeedc52ea8577598f3d6c8","observation_id":"5e0823fa-27d5-4f2a-a7f0-513413431ce6","resolution":{"observed_at":"2026-08-06T18:00:21.424555Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.410459Z","title":"Large language models can accurately predict searcher preferences","venue":null,"work_id":"5d558d11-66cd-4a9f-8060-32c75258647c","year":2023},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.274952Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:ecc4b45f6c2657b38c9b4234bea6bb977cf67cfb4cd14cf5a82e34ed65e71714","observation_id":"4d61a694-27ef-44c9-810a-e42925a020a8","resolution":{"observed_at":"2026-08-06T18:00:21.414114Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.398172Z","title":"Together inference api, 2024","venue":null,"work_id":"368a26e7-64ce-4ab9-997e-f320d1d9116d","year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.279139Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:fae441c53d160d4528a294a0ecb04b40d7b7c7440b2688810997ab976adf7ba4","observation_id":"c2aefca6-61c7-4556-b164-9a227c056da0","resolution":{"observed_at":"2026-08-06T18:00:21.401636Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.283508Z","title":"Umbrela: Umbrela is the (open-source reproduction of the) bing relevance assessor, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.283508Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:f7bfa93e056ecea6b3f057a0d1adfe6ab54b880774e9b5184ecc9ce20e5e3e13","observation_id":"90748b70-717d-4419-89ef-5e2d9ec030a8","resolution":{"observed_at":"2026-08-06T18:00:21.283508Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.379506Z","title":"Le, and Denny Zhou","venue":null,"work_id":"122f2ad9-3366-491b-8506-3e29ebed25aa","year":2022},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.286995Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:a7d7abdd511050a626947633509a90bc2c9a0777d07ab63147fafb19b265eec2","observation_id":"0ce41f22-2051-487c-9a9d-4412b58b58a2","resolution":{"observed_at":"2026-08-06T18:00:21.382904Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.367288Z","title":"Relevance judgment: What do information users consider beyond topicality? Journal of the American Society for Information Science & Technology, 57(7):961–973, May 2006","venue":null,"work_id":"91d567f6-64ef-4b34-8f61-cda595246522","year":2006},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.290488Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:725444a8970d9df0fbbb451c764df2ad514044865e4f241459ab0c310f1f5ff3","observation_id":"e80025c9-b9b7-4671-9f2d-d34d9e7ef1d8","resolution":{"observed_at":"2026-08-06T18:00:21.372243Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.356635Z","title":null,"venue":null,"work_id":"eb9d1269-be91-4761-9a4b-3d9de816d2b0","year":1941},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.294167Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:bc8054694d3791caf65a02652a8cb966ebbcf388025b2826b7e4462f4cd190ef","observation_id":"038180a4-47f7-4fad-a042-9ab6179fe786","resolution":{"observed_at":"2026-08-06T18:00:21.360604Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T18:00:21.343993Z","title":null,"venue":null,"work_id":"7b8a22eb-0ee9-4449-b7a9-885c105c93d9","year":2023},"citing_paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-06T18:00:21.298350Z"},"links":{"citing_paper":"/paper/2507.09488"},"observation_digest":"sha256:2409e87e7fe7b429bc6f166a97abf69b423496a7558461599b78213d8da92baa","observation_id":"7c063baa-5628-4e05-8ca8-3832eb1fd7bf","resolution":{"observed_at":"2026-08-06T18:00:21.349510Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2507.09488","last_updated":"2025-07-13T04:21:21Z","latest_version":1,"primary_category":"cs.IR","snapshot_observed_at":"2026-08-09T01:00:28.308638Z","submitted_at":"2025-07-13T04:21:21Z","title":"Criteria-Based LLM Relevance Judgments"},"reference_resolution":{"displayed":38,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":9,"verified_exact":0,"verified_fuzzy":29},"total_outbound_references":38},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"thesis":"As of 10 August 2026, this Paper Citation Record lists 38 of 38 outbound references and 5 inbound Pith citation observations for arXiv:2507.09488."}