{"as_of":"2026-08-15T14:53:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:e0d6acada176f587489b13f61be6bdecb6f22acef76ce700840d823281216745","coverage":[{"denominator":11,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":11,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-03T23:43:15.246548Z","state":"measured"},{"denominator":12,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":12,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-15T06:32:42.880941+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-06-28T07:16:30.725358Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-02T06:56:44.727065Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-13T07:47:38.831610Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"cited_work":{"arxiv_id":"2511.04355","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2511.04355","snapshot_observed_at":"2026-07-07T03:17:14.138901Z","title":"M., Heydari, M., Kazerooni, P., Maninger, D., and Mezini, M","venue":null,"work_id":"b49192ec-c1d4-4300-a810-5dd7f4758afe","year":null},"citing_paper":{"arxiv_id":"2606.09878","last_updated":"2026-06-03T01:28:00Z","snapshot_observed_at":"2026-07-06T23:49:08.412079Z","submitted_at":"2026-06-03T01:28:00Z","title":"FailureScope: Cross-Regime Behavioral Diagnosis of Language Model Weaknesses","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-06-28T07:16:30.725358Z"},"links":{"cited_paper":"/paper/2511.04355","citing_paper":"/paper/2606.09878"},"observation_digest":"sha256:4f88638c6b557a3b086e4629992d2f5f87f09ae6014b81dfefcff7b5bad99101","observation_id":"a62c1ff0-61b9-4a91-91ea-ba6bd33f0908","resolution":{"observed_at":"2026-07-07T03:17:14.138901Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2511.04355/citation-record","integrity":"/paper/2511.04355/integrity","json":"/paper/2511.04355/citation-record.json","paper":"/paper/2511.04355"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-03T23:43:14.703226Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-13T07:47:38.831610Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:14.703226Z"},"links":{"citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:431db267f0617f5e07af04f061e499b8595e411508c5255c33ed23816852bbd5","observation_id":"cad37e20-552c-414a-859e-82199c07ed36","resolution":{"observed_at":"2026-08-03T23:43:14.703226Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-03T23:43:14.735306Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-13T07:47:38.831610Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:14.735306Z"},"links":{"citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:3d3c0e3b7e90fb55110fefe8cd7e88f1ec49bd5db0dbd6786e5ec1f88817e903","observation_id":"b212e767-b003-45ec-9d31-97f35e21915d","resolution":{"observed_at":"2026-08-03T23:43:14.735306Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T11:58:24.516369Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-03T23:43:14.786547Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-13T07:47:38.831610Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:14.786547Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:ba0fc94a9e5b8654a0d16c5190fb03e75adfeeeb8c266e764fc2c196e61d4563","observation_id":"d89f27f7-d5a2-4b29-b2a9-4479aab96e78","resolution":{"observed_at":"2026-08-03T23:43:14.786547Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2108.07732","last_updated":"2021-08-16T03:57:30Z","snapshot_observed_at":"2026-08-02T19:23:53.535075Z","submitted_at":"2021-08-16T03:57:30Z","title":"Program Synthesis with Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.07732","snapshot_observed_at":"2026-08-03T23:43:14.853124Z","title":"Austin, A","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-13T07:47:38.831610Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:14.853124Z"},"links":{"cited_paper":"/paper/2108.07732","citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:ce4e2315b9fc2c8070cb6cd22e6a82be72f231bee198b88d34fd3bd31df62f63","observation_id":"a8f4ff59-3277-4774-950d-ec6952061c44","resolution":{"observed_at":"2026-08-03T23:43:14.853124Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.00699","last_updated":"2025-07-09T19:56:02Z","snapshot_observed_at":"2026-08-13T00:40:34.368813Z","submitted_at":"2024-03-31T14:32:02Z","title":"A Comprehensive Survey of Contamination Detection Methods in Large Language Models","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.00699","snapshot_observed_at":"2026-08-03T23:43:14.882558Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-13T07:47:38.831610Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:14.882558Z"},"links":{"cited_paper":"/paper/2404.00699","citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:80ca8d71db14c64d45ca315fc723639516c5188f67a00e34a3731ce2136efb2c","observation_id":"b511eb66-97da-4e5a-9526-7986e81dd6df","resolution":{"observed_at":"2026-08-03T23:43:14.882558Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.15877","last_updated":"2025-04-01T08:36:44Z","snapshot_observed_at":"2026-08-14T15:45:30.011625Z","submitted_at":"2024-06-22T15:52:04Z","title":"BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.15877","snapshot_observed_at":"2026-08-03T23:43:14.940179Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-13T07:47:38.831610Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:14.940179Z"},"links":{"cited_paper":"/paper/2406.15877","citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:d53576609eb739ba42eb2d213fbe54371ba8ddb9e7bd20a97806dbbd39d6b1b4","observation_id":"0f02549c-e2cb-47f2-ad33-7dd4011b3610","resolution":{"observed_at":"2026-08-03T23:43:14.940179Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.06770","last_updated":"2024-11-11T23:05:04Z","snapshot_observed_at":"2026-08-12T14:56:35.025839Z","submitted_at":"2023-10-10T16:47:29Z","title":"SWE-bench: Can Language Models Resolve Real-World GitHub Issues?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.06770","snapshot_observed_at":"2026-08-03T23:43:14.983161Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-13T07:47:38.831610Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:14.983161Z"},"links":{"cited_paper":"/paper/2310.06770","citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:4677b870b0d2ba7020d7c952a71dd0fe4f647260f26ce8e2d6a57a75bb481f93","observation_id":"56be89f8-b821-4f69-b6f4-d5f5a7191cdb","resolution":{"observed_at":"2026-08-03T23:43:14.983161Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-03T23:43:15.074745Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-13T07:47:38.831610Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:15.074745Z"},"links":{"citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:d7c276710827e24dfc6c02f09b7391643ba0229a47a54f88ac277d44e9099ec6","observation_id":"e4ee9963-cdfa-4185-a2a3-c30eee9424e3","resolution":{"observed_at":"2026-08-03T23:43:15.074745Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.21199","last_updated":"2024-12-31T08:20:42Z","snapshot_observed_at":"2026-08-12T23:35:38.017550Z","submitted_at":"2024-12-30T18:58:58Z","title":"HumanEval Pro and MBPP Pro: Evaluating Large Language Models on Self-invoking Code Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.21199","snapshot_observed_at":"2026-08-03T23:43:15.114501Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-13T07:47:38.831610Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:15.114501Z"},"links":{"cited_paper":"/paper/2412.21199","citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:7c8c27659e80121ec299093c8e2af860e4ce153ad916db972f7d469f724b5aa0","observation_id":"3b213d7f-3344-4e7e-9e2a-b143a710c997","resolution":{"observed_at":"2026-08-03T23:43:15.114501Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2105.09938","last_updated":"2021-11-08T21:16:44Z","snapshot_observed_at":"2026-08-14T15:19:05.440904Z","submitted_at":"2021-05-20T17:58:42Z","title":"Measuring Coding Challenge Competence With APPS","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2105.09938","snapshot_observed_at":"2026-08-03T23:43:15.170191Z","title":"Hendrycks, S","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-13T07:47:38.831610Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:15.170191Z"},"links":{"cited_paper":"/paper/2105.09938","citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:3a12cad728d6ac38fe2bceef46e7bbdddc09361decbdb7cf8689ece4d266ae88","observation_id":"9f913bb4-fb57-465d-95e6-93f62a46e99f","resolution":{"observed_at":"2026-08-03T23:43:15.170191Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-03T23:43:15.246548Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","snapshot_observed_at":"2026-08-13T07:47:38.831610Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-03T23:43:15.246548Z"},"links":{"citing_paper":"/paper/2511.04355"},"observation_digest":"sha256:769978e7fec10df3afcda21376268d4baeb37c733106099179156af5c6d80cfe","observation_id":"ab594daf-a75d-4197-8369-128f0ea6b1d5","resolution":{"observed_at":"2026-08-03T23:43:15.246548Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2511.04355","last_updated":"2025-11-06T13:38:03Z","latest_version":1,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-13T07:47:38.831610Z","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks"},"reference_resolution":{"displayed":11,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":11,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":11},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"thesis":"As of 15 August 2026, this Paper Citation Record lists 11 of 11 outbound references and 1 inbound Pith citation observation for arXiv:2511.04355."}