{"as_of":"2026-08-22T05:50:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:7e2a69fc8f29b6233e0ff9fd404df6bcd18acbaf735407c2d57b6117c17f69e7","coverage":[{"denominator":27,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":27,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-08T04:17:31.943657Z","state":"measured"},{"denominator":27,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":27,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-21T06:32:19.484+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2608.02996/citation-record","integrity":"/paper/2608.02996/integrity","json":"/paper/2608.02996/citation-record.json","paper":"/paper/2608.02996"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2507.19457","last_updated":"2026-02-14T11:42:30Z","snapshot_observed_at":"2026-08-14T13:34:39.415942Z","submitted_at":"2025-07-25T17:42:32Z","title":"GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2507.19457","snapshot_observed_at":"2026-08-08T04:17:31.686300Z","title":"Agrawal et al","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.686300Z"},"links":{"cited_paper":"/paper/2507.19457","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:94b90d12ae01b8fd8310725fa72b0cca80c4c644fcc8dbe31f5862f1d756631b","observation_id":"e3030bc4-0f75-445e-973a-35afed1ec6df","resolution":{"observed_at":"2026-08-08T04:17:31.686300Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.854116Z","title":"Artificial intelligence governance in health systems: Systematic review of frameworks and integrative model proposal.J","venue":null,"work_id":"103babcc-fe53-4994-b276-8aaa0c68be5f","year":2026},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.697353Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:4197b6597ea65e92c28ccb72861a855e5b07e2ded2c60f2eb9411e4a826928ef","observation_id":"4674ccfd-5493-4e6f-9107-4215eb480aa9","resolution":{"observed_at":"2026-08-08T04:17:32.861169Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.823060Z","title":"Bender, Timnit Gebru, Angelina McMillan-Major, and Shmargi Shmitchell","venue":null,"work_id":"ed53a2fd-f9bd-48ce-808b-c0b93a1a5957","year":2021},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.708073Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:6fbdbb7d09b56217b4256be53dd7bc56496ccdf6dd49cb673602e5fb25372c44","observation_id":"09513cd2-de6c-49ec-82b0-135719d40440","resolution":{"observed_at":"2026-08-08T04:17:32.830022Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2108.07258","last_updated":"2022-07-12T23:45:14Z","snapshot_observed_at":"2026-08-02T09:20:40.804790Z","submitted_at":"2021-08-16T17:50:08Z","title":"On the Opportunities and Risks of Foundation Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.07258","snapshot_observed_at":"2026-08-08T04:17:31.716370Z","title":"On the opportunities and risks of foundation models.arXiv preprint arXiv:2108.07258, 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.716370Z"},"links":{"cited_paper":"/paper/2108.07258","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:702dfc5b1c1f2caa1592f6a153292ebd19d1fb2f908b315d1fe9256100216599","observation_id":"8ebf4aa4-c6dd-4a47-9bae-b0e95439821d","resolution":{"observed_at":"2026-08-08T04:17:31.716370Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.789472Z","title":"Sustainability and participation in the digital commons.Interactions, 2017","venue":null,"work_id":"009bb7c5-f72f-497f-9d2b-74e909f6efbc","year":2017},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.726929Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:839f203bca9ecb3d9c03ef591475d97a3b18a98864156eb9ca43a0ec1dcb94cf","observation_id":"1e51c863-8253-40e5-bb5f-4883145be71b","resolution":{"observed_at":"2026-08-08T04:17:32.800347Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1511.04970","last_updated":"2017-02-06T00:51:34Z","snapshot_observed_at":"2026-08-16T14:01:46.750647Z","submitted_at":"2015-11-16T14:29:38Z","title":"Learning about Spanish dialects through Twitter","version":2},"cited_work":{"arxiv_id":"1511.04970","doi":null,"metadata_source":"pith","pith_arxiv_id":"1511.04970","snapshot_observed_at":"2026-08-08T04:17:32.291501Z","title":"Learning about Spanish dialects through Twitter","venue":"stat.ML","work_id":"992d58b4-b955-4e6c-9e2c-473ea8295c4c","year":2015},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.734956Z"},"links":{"cited_paper":"/paper/1511.04970","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:3d71cb12e0934df9a990ba83c5356b7b89900484c7ad0f205c18cdd459cd9e4d","observation_id":"642d7662-9159-467b-bed3-0b8781df20e7","resolution":{"observed_at":"2026-08-08T04:17:32.300809Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.754136Z","title":"Evaluation gaps in machine learning practice","venue":null,"work_id":"a7b3d0a3-e2ff-4f7c-838f-92d834df9a69","year":2022},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.744217Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:61f1d65bb35ef62a56a9ae87d4b26ee2365059d9f636de18a7e39474242bb0d0","observation_id":"0eeaf8ff-291c-43df-8f9a-739a0b6774a0","resolution":{"observed_at":"2026-08-08T04:17:32.774089Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.727358Z","title":"Epistemic injustice in generative ai","venue":null,"work_id":"8b4d7615-8f04-455f-91f0-2f9f17923be4","year":2024},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.752593Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:40ee023db10d98e1e63e2532b7761ea33a8695620a527fe2f84b0045c68d1880","observation_id":"fc954717-f020-474a-8ae5-f022f5587956","resolution":{"observed_at":"2026-08-08T04:17:32.734450Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.03714","last_updated":"2023-10-05T17:37:25Z","snapshot_observed_at":"2026-08-04T05:21:05.846165Z","submitted_at":"2023-10-05T17:37:25Z","title":"DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.03714","snapshot_observed_at":"2026-08-08T04:17:31.763497Z","title":"Dspy: Compiling declarative language model calls into self-improving pipelines.arXiv preprint arXiv:2310.03714, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.763497Z"},"links":{"cited_paper":"/paper/2310.03714","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:2aba4ed915fb1232230110d8f683a789cd9cc55ecf251a8e0c5a4add755563cf","observation_id":"749db3e3-648b-4a62-b588-c4b172907feb","resolution":{"observed_at":"2026-08-08T04:17:31.763497Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.701813Z","title":"Latamgpt: Open llms for latin american spanish.huggingface.co/ latam-gpt, 2025","venue":null,"work_id":"5760f6e7-6358-4c50-8e6a-43e553c50e62","year":2025},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.774100Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:3d33439e84d5ba3a1edc00d42715794129769e42b616a9d0a324f45146fdef76","observation_id":"e0259c74-9d9f-4482-87e0-d7d2c22697a1","resolution":{"observed_at":"2026-08-08T04:17:32.709645Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.670239Z","title":"Some simple economics of open source.The Journal of Industrial Economics, 50(2):197–234, 2002","venue":null,"work_id":"5d4d27ec-a6db-475b-b62c-42966f350134","year":2002},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.785001Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:b63886a04d880ca5f9b56fea1082ae39f796e57c2e960e596e5e01f46c827988","observation_id":"53d18846-a795-4a20-b919-34433a02ff3b","resolution":{"observed_at":"2026-08-08T04:17:32.676436Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-08T04:17:31.793770Z","title":"Holistic evaluation of language models.arXiv preprint arXiv:2211.09110, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.793770Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:4d9a8a1d61b66cb5379c59703f75455fac986ea54951e8547c21fb91837e65f0","observation_id":"b4feaab7-b720-46eb-8c3f-60db73e005aa","resolution":{"observed_at":"2026-08-08T04:17:31.793770Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.640214Z","title":"The medical algorithmic audit.The Lancet Digital Health, 4(3):e152–e163, 2022","venue":null,"work_id":"1a536b2a-612f-4fb3-b451-159e7eebc606","year":2022},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.801431Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:2e4e055e986ac3a0ce3e234dcf6b1b34f15dd908d60d1b6873b32f1e94a10231","observation_id":"c6aec451-e7ac-4ae2-9d1d-7639278a9c92","resolution":{"observed_at":"2026-08-08T04:17:32.649175Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.02680","last_updated":"2024-10-05T18:20:58Z","snapshot_observed_at":"2026-08-16T14:21:36.284839Z","submitted_at":"2024-02-05T02:32:09Z","title":"Large Language Models are Geographically Biased","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.02680","snapshot_observed_at":"2026-08-08T04:17:31.810090Z","title":"Lobell, and Stefano Ermon","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.810090Z"},"links":{"cited_paper":"/paper/2402.02680","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:771cc40df174360a107b6450303b3015f1d620347d56e1087675d75d60e7c83b","observation_id":"42b63746-05fd-4fbb-be3d-fb1512eee4ae","resolution":{"observed_at":"2026-08-08T04:17:31.810090Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.614183Z","title":"Robertson","venue":null,"work_id":"2f0abd1a-ced8-40f9-8536-ba6a9973e0fe","year":2021},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.826397Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:d8af18656700648c2d5d8c99ac6b92f75b9dc7092e5e9b9d6770fe4414a3b689","observation_id":"95b7657d-2e2e-4fff-b57a-b316fd25811b","resolution":{"observed_at":"2026-08-08T04:17:32.622604Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.585755Z","title":"Auditing large language models: a three-layered approach.AI and Ethics, 2023","venue":null,"work_id":"d0013735-aa13-45df-8140-a7d9fa14bbea","year":2023},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.833689Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:aac5aab749f956665b9f597555a783d462f92676e2656c5340557041833c8e57","observation_id":"fb26bcfd-4787-4e08-a0aa-1068a7ebbb5b","resolution":{"observed_at":"2026-08-08T04:17:32.595137Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.11695","last_updated":"2024-10-06T17:34:26Z","snapshot_observed_at":"2026-08-20T16:46:21.676803Z","submitted_at":"2024-06-17T16:12:03Z","title":"Optimizing Instructions and Demonstrations for Multi-Stage Language Model Programs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.11695","snapshot_observed_at":"2026-08-08T04:17:31.840402Z","title":"Optimizing instructions and demonstrations for multi-stage language model programs","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.840402Z"},"links":{"cited_paper":"/paper/2406.11695","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:eec6284d7d34ebe19a551c5945b2ef6efbb6b3b9a0d583591ec55d9e7c230980","observation_id":"e4a1719a-3cac-4952-b079-947b6add83a4","resolution":{"observed_at":"2026-08-08T04:17:31.840402Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2003.12206","last_updated":"2020-12-30T21:32:34Z","snapshot_observed_at":"2026-08-19T19:54:57.424740Z","submitted_at":"2020-03-27T02:16:25Z","title":"Improving Reproducibility in Machine Learning Research (A Report from the NeurIPS 2019 Reproducibility Program)","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2003.12206","snapshot_observed_at":"2026-08-08T04:17:31.852636Z","title":"Improving reproducibility in machine learning research (a report from the neurips 2019 reproducibility program).JMLR, 2020","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.852636Z"},"links":{"cited_paper":"/paper/2003.12206","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:a6700ec83c86eaba046d279e317717337a4323c25358272e0e9c1ebef6e8141b","observation_id":"19c159ac-e7ef-4e43-849b-796c82b9d614","resolution":{"observed_at":"2026-08-08T04:17:31.852636Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.555452Z","title":"A governance model for the application of ai in health care.J","venue":null,"work_id":"0eb6ee66-cb88-487c-a379-a83e7a684daa","year":2019},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.870231Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:1d03e9a15f4efa3b2cf43068efe97adda38b1a1e3b43722291be56da164f652f","observation_id":"db40ef7b-2e12-48af-92ea-ef1df90eab98","resolution":{"observed_at":"2026-08-08T04:17:32.563111Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.12990","last_updated":"2024-11-20T02:38:24Z","snapshot_observed_at":"2026-08-19T08:26:44.789995Z","submitted_at":"2024-11-20T02:38:24Z","title":"BetterBench: Assessing AI Benchmarks, Uncovering Issues, and Establishing Best Practices","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.12990","snapshot_observed_at":"2026-08-08T04:17:31.890320Z","title":"Betterbench: Assessing ai benchmarks, uncovering issues, and establishing best practices.arXiv preprint arXiv:2411.12990, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.890320Z"},"links":{"cited_paper":"/paper/2411.12990","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:10ac4e496d1753e70ee95ae0e30417a0afcd535d52b27f352b4f5c14992a2dea","observation_id":"30a49794-cdea-44d1-9de1-eee8fddb6218","resolution":{"observed_at":"2026-08-08T04:17:31.890320Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.518991Z","title":"Prompt programming for large language models: Beyond the few-shot paradigm","venue":null,"work_id":"49830bd2-25b0-4130-a8f2-4bb93e2209c8","year":2021},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.896304Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:49ee41a123dc18e2eca0234d25b849740ea0bfe4e8274c0aabf64288bafdf553","observation_id":"cbcbb797-b100-4c59-9a25-985ce7039c67","resolution":{"observed_at":"2026-08-08T04:17:32.528048Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.486494Z","title":"Digital sovereignty and artificial intelligence: a normative approach.AI and Ethics, 2024","venue":null,"work_id":"b2f1234f-01f7-4f61-8472-94a726781d82","year":2024},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.906077Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:896c0c379548bfbc2ce02ff8e6ca9b2996539da9321bccac062cab0fa0bccb39","observation_id":"e66aa136-350d-47d2-8231-2b6f8c5697fe","resolution":{"observed_at":"2026-08-08T04:17:32.496417Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.450786Z","title":null,"venue":null,"work_id":"9b0ef418-93da-4079-9180-50ddd438921f","year":2020},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.912582Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:db1e472c8bfe8c486fa2de79621658f975da965bf2102c12f174eca1dbcca481","observation_id":"04d392bc-fb5d-45ff-b7ff-db25917f4cbe","resolution":{"observed_at":"2026-08-08T04:17:32.461929Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.419351Z","title":"Ethics of ai and cybersecurity when sovereignty is at stake.Minds and Machines, 2019","venue":null,"work_id":"ca5bfc68-2db3-49c9-afc2-f08538935706","year":2019},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.920536Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:48015001e4330556e12d76fa247f8d30083273ad9bd61cac664e6e04f62710b2","observation_id":"8aa9fb68-3c7e-4135-989d-288471c6ad87","resolution":{"observed_at":"2026-08-08T04:17:32.427949Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T04:17:32.389954Z","title":"Weber et al","venue":null,"work_id":"40b8b7f1-352b-49cf-8926-ec2d6757e2c6","year":2019},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.929240Z"},"links":{"citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:ce9cb0916afdb69af8da1e01e37b95810595940c4c4d77df7eeb1dcc1afa7455","observation_id":"4ede66d8-890b-45d7-92ef-18c19eb412bd","resolution":{"observed_at":"2026-08-08T04:17:32.402349Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.04244","last_updated":"2024-06-06T16:41:39Z","snapshot_observed_at":"2026-08-17T14:15:23.870937Z","submitted_at":"2024-06-06T16:41:39Z","title":"Benchmark Data Contamination of Large Language Models: A Survey","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.04244","snapshot_observed_at":"2026-08-08T04:17:31.936704Z","title":"Benchmark data contamination of large language models: A survey.arXiv preprint arXiv:2406.04244, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.936704Z"},"links":{"cited_paper":"/paper/2406.04244","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:81beb0a0a3f2bf8ccd31cf13c65d59de4399b045929421e21fc5a5c20a9a79e7","observation_id":"f7059a58-e641-411b-a13a-f79dd2a3bfea","resolution":{"observed_at":"2026-08-08T04:17:31.936704Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.05685","last_updated":"2023-12-24T02:01:34Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-06-09T05:55:52Z","title":"Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.05685","snapshot_observed_at":"2026-08-08T04:17:31.943657Z","title":"Judging llm-as-a-judge with mt-bench and chatbot arena","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-08T04:17:31.943657Z"},"links":{"cited_paper":"/paper/2306.05685","citing_paper":"/paper/2608.02996"},"observation_digest":"sha256:aa420339becd3e31e2e8c3dadada0ad9131b2302814605c5f94da9afbbf5aea8","observation_id":"81c84ca6-74dd-4929-91d6-7f98a1b65a2f","resolution":{"observed_at":"2026-08-08T04:17:31.943657Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2608.02996","last_updated":"2026-08-04T01:26:03Z","latest_version":1,"primary_category":"cs.AI","snapshot_observed_at":"2026-08-17T09:03:49.254861Z","submitted_at":"2026-08-04T01:26:03Z","title":"On the missing benchmarks layer and a potential solution"},"reference_resolution":{"displayed":27,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":11,"verified_exact":1,"verified_fuzzy":15},"total_outbound_references":27},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"thesis":"As of 22 August 2026, this Paper Citation Record lists 27 of 27 outbound references and 0 inbound Pith citation observations for arXiv:2608.02996."}