{"as_of":"2026-08-15T07:30:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:178f42e4c9110107482e63ae771550de1153831885016f609544b05f25a6215a","coverage":[{"denominator":45,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":45,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T16:38:42.962606Z","state":"measured"},{"denominator":46,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":46,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-15T06:32:42.880941+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-10T20:20:21.052795Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"pith","source_observed_at":"2026-08-11T00:16:15.729268Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"cited_work":{"arxiv_id":"2411.13323","doi":"10.48550/arxiv.2411.13323","metadata_source":"pith","pith_arxiv_id":"2411.13323","snapshot_observed_at":"2026-08-11T00:16:15.729268Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","venue":"cs.SE","work_id":"58940c65-7961-40bd-ad15-3eb2fd8b6629","year":2024},"citing_paper":{"arxiv_id":"2501.08840","last_updated":"2025-01-15T14:50:46Z","snapshot_observed_at":"2026-08-14T03:12:43.720121Z","submitted_at":"2025-01-15T14:50:46Z","title":"CveBinarySheet: A Comprehensive Pre-built Binaries Database for IoT Vulnerability Analysis","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-10T20:20:21.052795Z"},"links":{"cited_paper":"/paper/2411.13323","citing_paper":"/paper/2501.08840"},"observation_digest":"sha256:b29bc45892a72c194f8202814a4bcf65427924c862ad6eeea9b0df60f98b2f98","observation_id":"904278f3-d300-46fa-94f9-2b1ea3fc9164","resolution":{"observed_at":"2026-08-10T20:20:21.116359Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2411.13323/citation-record","integrity":"/paper/2411.13323/integrity","json":"/paper/2411.13323/citation-record.json","paper":"/paper/2411.13323"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:44.172401Z","title":"A survey on software fault localization,","venue":null,"work_id":"12302fcf-9ad4-4efa-8806-4aa9ced7e424","year":2016},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.252494Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:c281ade62caf5470f0672b62d0d0f4621df1ddb06bbac8f6fbeb48378dc78ce7","observation_id":"19cccd37-eae3-4fe3-9883-69a6b40c60dc","resolution":{"observed_at":"2026-08-12T16:38:44.177178Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:44.159488Z","title":"Automated Program Repair,","venue":null,"work_id":"85a01ede-f6d6-4fb7-8f2f-fa52c20cbd17","year":2019},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.256974Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:bfb66914ad5d555fbfc083ccd350f69a1891f765b38dde870edd01212080cfc2","observation_id":"619004af-2b05-49c6-aad8-7356b5ef0525","resolution":{"observed_at":"2026-08-12T16:38:44.163595Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:44.147236Z","title":"Defects4j: a database of existing faults to enable controlled testing studies for java programs,","venue":null,"work_id":"1341c7f9-68af-44bc-9420-034f5dc0f3f2","year":2014},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.261686Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:ddffa2f552dbe5c810da74707c967b4279ed65f48d3285ee4f1bb33a2cc01433","observation_id":"9546f144-40e9-40a9-be7a-788c08e8949b","resolution":{"observed_at":"2026-08-12T16:38:44.151216Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:44.134494Z","title":"Bugsinpy: a database of existing bugs in python programs to enable controlled testing and debugging studies,","venue":null,"work_id":"369f5c1a-fb9a-4e60-a22f-ee5fac78b2c2","year":2020},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.265987Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:f8e5ef246c0e3df124386a5311ad1d11a9084b5d1a65d8ce27503bd8932e26d4","observation_id":"a52e983f-e0fe-4a17-b62e-0a3a1e26410c","resolution":{"observed_at":"2026-08-12T16:38:44.139126Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:44.063979Z","title":"SWE-bench: Can language models resolve real-world github issues?","venue":null,"work_id":"5342ead1-427c-4332-9c58-057faacf3196","year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.270348Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:cc48354f1832e13893138f675a5bc02502c828e9bd9a9431ced18b08d08614bb","observation_id":"d0bfa93d-fe52-4f03-8c37-1e0cf92ad63f","resolution":{"observed_at":"2026-08-12T16:38:44.124917Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.16898","last_updated":"2024-03-28T05:00:47Z","snapshot_observed_at":"2026-08-14T10:47:53.731739Z","submitted_at":"2024-03-25T16:10:25Z","title":"Concerned with Data Contamination? Assessing Countermeasures in Code Language Model","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.16898","snapshot_observed_at":"2026-08-12T16:38:42.274919Z","title":"Concerned with data contamination? assessing countermeasures in code language model,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.274919Z"},"links":{"cited_paper":"/paper/2403.16898","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:fdd02684512e5c37e62880d93eea239b2826673f0d26ae8618e12c7e8d49ae9d","observation_id":"05131f45-ffc5-4890-a61f-3b941cf05c6b","resolution":{"observed_at":"2026-08-12T16:38:42.274919Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2207.07048","last_updated":"2022-07-14T16:44:59Z","snapshot_observed_at":"2026-08-14T14:36:33.379781Z","submitted_at":"2022-07-14T16:44:59Z","title":"Leakage and the Reproducibility Crisis in ML-based Science","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2207.07048","snapshot_observed_at":"2026-08-12T16:38:42.280170Z","title":"Leakage and the reproducibility crisis in ml-based science,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.280170Z"},"links":{"cited_paper":"/paper/2207.07048","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:c979f5e2f9844f051d6fd4cbc6ec8c3394bc0ceb8b07de56e50b571d4099f3ae","observation_id":"e1c4d370-6e1e-4cf5-bcab-de488e646eed","resolution":{"observed_at":"2026-08-12T16:38:42.280170Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.18824","last_updated":"2024-04-29T16:05:36Z","snapshot_observed_at":"2026-08-13T04:56:39.486630Z","submitted_at":"2024-04-29T16:05:36Z","title":"Benchmarking Benchmark Leakage in Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.18824","snapshot_observed_at":"2026-08-12T16:38:42.284289Z","title":"Benchmarking benchmark leakage in large language models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.284289Z"},"links":{"cited_paper":"/paper/2404.18824","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:38ade4985e7b72b162dd7c554416ccd06f2319e0e94b3d6a3bf0dfbfc0b8d5e1","observation_id":"91903aa1-6de5-4571-be83-ce728ec2225b","resolution":{"observed_at":"2026-08-12T16:38:42.284289Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.10677","last_updated":"2023-09-27T01:15:49Z","snapshot_observed_at":"2026-08-13T10:09:54.548080Z","submitted_at":"2023-09-19T15:02:58Z","title":"Estimating Contamination via Perplexity: Quantifying Memorisation in Language Model Evaluation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.10677","snapshot_observed_at":"2026-08-12T16:38:42.288484Z","title":"Estimating contamination via perplexity: Quantifying memo- risation in language model evaluation,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.288484Z"},"links":{"cited_paper":"/paper/2309.10677","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:ce44a04cfec390819d782eb511956486d89b7a44bccfc8f00a91ecfaae665879","observation_id":"4f9e806b-50c5-4845-a892-601a9d8ad91c","resolution":{"observed_at":"2026-08-12T16:38:42.288484Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:42.296648Z","title":"Bugsc++: A highly usable real world defect benchmark for c/c++,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.296648Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:590e7d8c8e5a3b0e670da5df8eba2474b3df6da7b36314e7eb71f7c8c19d7526","observation_id":"9cdb5a41-7298-4f92-9321-46d76c4fce2f","resolution":{"observed_at":"2026-08-12T16:38:42.296648Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.884096Z","title":"Gitbug-java: A reproducible benchmark of recent java bugs,","venue":null,"work_id":"cf1aae2f-fe5c-4598-a38c-a6a0a06d3225","year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.300069Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:98e0e06f591dc92f283591d06120bff88120c5525cf825ad8f5562f4ca872aa1","observation_id":"d09c4910-7504-420a-9664-9fc61b9e9c9c","resolution":{"observed_at":"2026-08-12T16:38:43.959940Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.01489","last_updated":"2024-10-29T17:29:27Z","snapshot_observed_at":"2026-08-10T19:28:41.964638Z","submitted_at":"2024-07-01T17:24:45Z","title":"Agentless: Demystifying LLM-based Software Engineering Agents","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.01489","snapshot_observed_at":"2026-08-12T16:38:42.304045Z","title":"Agentless: Demystifying llm-based software engineering agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.304045Z"},"links":{"cited_paper":"/paper/2407.01489","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:a4c83ca80d03e96de2256dde15566503efad35b8906f5c354c74eb93aaaac0e3","observation_id":"7fb3822f-891b-4631-97b0-6ce0bed8fcee","resolution":{"observed_at":"2026-08-12T16:38:42.304045Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.16741","last_updated":"2025-04-18T18:14:31Z","snapshot_observed_at":"2026-08-14T17:59:52.524740Z","submitted_at":"2024-07-23T17:50:43Z","title":"OpenHands: An Open Platform for AI Software Developers as Generalist Agents","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.16741","snapshot_observed_at":"2026-08-12T16:38:42.308840Z","title":"Openhands: An open platform for ai software developers as generalist agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.308840Z"},"links":{"cited_paper":"/paper/2407.16741","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:39923dd8681d59f3a8ee4236eaf783f6c8940366dc25fe87c73b7d63186e1427","observation_id":"9db61581-7331-4858-9b14-7e0d47a8eb4c","resolution":{"observed_at":"2026-08-12T16:38:42.308840Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.06992","last_updated":"2024-10-10T13:13:09Z","snapshot_observed_at":"2026-08-12T22:27:08.380398Z","submitted_at":"2024-10-09T15:38:53Z","title":"SWE-Bench+: Enhanced Coding Benchmark for LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.06992","snapshot_observed_at":"2026-08-12T16:38:42.349704Z","title":"Swe-bench+: Enhanced coding benchmark for llms,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.349704Z"},"links":{"cited_paper":"/paper/2410.06992","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:631674c8918a49d8e579aab4c794eb9b516ba1d5ea7abb0c1a806ecdb44ba51b","observation_id":"734c1bc4-4623-40cb-aa05-49b7d4d376e7","resolution":{"observed_at":"2026-08-12T16:38:42.349704Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.872184Z","title":"On the resemblance and containment of documents,","venue":null,"work_id":"a11073f1-cb94-44d1-a650-0ccc895eceb2","year":1997},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.405481Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:3c23d7d98ab88a7fcf38566a34eb43d916f9e5baca7420a1483367154ac9a363","observation_id":"c6e344e4-9bad-4ae1-9a67-b4b9258ae866","resolution":{"observed_at":"2026-08-12T16:38:43.877092Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.859668Z","title":"Similarity search in high dimensions via hashing,","venue":null,"work_id":"7ea8af59-ff7a-4253-9056-3d33bef456ab","year":1999},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.506527Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:818bf646fd469043dcb550167fec39131ea1b5ab74c66c9f794e5c3cf0a4519d","observation_id":"6242c7ba-190b-4661-ac2c-3fde315d8ec7","resolution":{"observed_at":"2026-08-12T16:38:43.863930Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.847687Z","title":"Codegen: An open large language model for code with multi-turn program synthesis,","venue":null,"work_id":"7894a460-4271-4d58-a989-f8b0a026af3f","year":2023},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.510429Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:eb41bd097242f9ed47fea6b3ca379cf4f6eb84a2d637f3b5139ea69b480fac1c","observation_id":"5233ba2b-6590-4991-bb27-c1ab79d191d5","resolution":{"observed_at":"2026-08-12T16:38:43.852055Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.833545Z","title":"Code llama: Open foundation models for code,","venue":null,"work_id":"a2307b40-eb96-42de-8bac-09c04736d517","year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.514409Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:12a551280d2aadc311baed40debdae4f817eecfcebdc4a86c2bb0c6c425d982b","observation_id":"fcf00b6b-9882-4094-a865-574eb4dff3d6","resolution":{"observed_at":"2026-08-12T16:38:43.837616Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:42.522310Z","title":"Llama: Open and efficient foundation language models,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.522310Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:15c02c4aee4056294d513c183c097af6053d67a8801cbe7e03344162dbbe78a2","observation_id":"5722db26-a0a7-4e0f-b402-92c10b750856","resolution":{"observed_at":"2026-08-12T16:38:42.522310Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.12950","last_updated":"2024-01-31T19:47:26Z","snapshot_observed_at":"2026-08-13T04:25:38.282910Z","submitted_at":"2023-08-24T17:39:13Z","title":"Code Llama: Open Foundation Models for Code","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.12950","snapshot_observed_at":"2026-08-12T16:38:42.518109Z","title":"Available: https://doi.org/10.48550/arXiv.2308.12950","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.518109Z"},"links":{"cited_paper":"/paper/2308.12950","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:94bf601a9172b614d16c2f4b3078fcf20a1b87c82c1e469c650e9c359526a176","observation_id":"6b7abd74-868e-4562-a69b-ed465abbc3a7","resolution":{"observed_at":"2026-08-12T16:38:42.518109Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.00118","last_updated":"2024-10-02T15:22:49Z","snapshot_observed_at":"2026-08-02T16:20:09.773989Z","submitted_at":"2024-07-31T19:13:07Z","title":"Gemma 2: Improving Open Language Models at a Practical Size","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.00118","snapshot_observed_at":"2026-08-12T16:38:42.530024Z","title":"Gemma 2: Improving open language models at a practical size,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.530024Z"},"links":{"cited_paper":"/paper/2408.00118","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:a08bcd8934c8493e183fff65c700c53448fb32702df40496bf9c48167ade43c7","observation_id":"5648e50f-7837-4773-8691-f79a815bf547","resolution":{"observed_at":"2026-08-12T16:38:42.530024Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.694587Z","title":"Starcoder 2 and the stack v2: The next generation,","venue":null,"work_id":"c5aec53f-3b40-4c09-b336-98141aed9321","year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.525883Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:8b106b6f52651322ad8d6664b3fa3afe0e138178a46646bff8805200544aef49","observation_id":"6a8e0c35-14a6-47d8-9314-ecbfd02df6bf","resolution":{"observed_at":"2026-08-12T16:38:43.776154Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.06825","last_updated":"2023-10-10T17:54:58Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-10-10T17:54:58Z","title":"Mistral 7B","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.06825","snapshot_observed_at":"2026-08-12T16:38:42.758008Z","title":"Mistral 7b,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.758008Z"},"links":{"cited_paper":"/paper/2310.06825","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:08342fb07956b2643493022543a90e8800b9fa71239cc701977f9c1a0d731c92","observation_id":"c97a4665-f0e4-47f3-a172-c052093eb1fa","resolution":{"observed_at":"2026-08-12T16:38:42.758008Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.682270Z","title":"Codegemma: Open code models based on gemma,","venue":null,"work_id":"ccbefe3b-c11e-4e78-9a59-40aefa98e931","year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.586465Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:87580e74c12155790dfcae3bc8afb0b848ea53bdf2c297a57cfa3c6233a2c7f6","observation_id":"29f57848-3c77-446d-ac1a-2e9bf18ee480","resolution":{"observed_at":"2026-08-12T16:38:43.686661Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.656119Z","title":"Repairllama: Efficient representations and fine-tuned adapters for program repair,","venue":null,"work_id":"ce05012c-1402-450d-a733-11c40dee1c34","year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.795695Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:956a89e7a6396398c854c7049d5e06d7525347fcd2004290fdf087a8ccd16557","observation_id":"5f6b8255-4788-4973-bdc5-e68bb0434e98","resolution":{"observed_at":"2026-08-12T16:38:43.660234Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.642785Z","title":"Large language model for vulnerability detection: Emerging results and future directions,","venue":null,"work_id":"784215a1-8530-47a3-9ab5-33b8f35c2a10","year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.804097Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:c45cc82fc1f7addbc04e8f1794e90868d35bd56e8010f51b8f24230710ce259e","observation_id":"1a7cc93e-7ad8-43a2-8bcb-3823f9b331c6","resolution":{"observed_at":"2026-08-12T16:38:43.647047Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.668944Z","title":"Large language models for test-free fault localization,","venue":null,"work_id":"55b9921b-2e14-4ffa-b548-686d8ee71c27","year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.791692Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:e57de52d7c97af7d926da0d554999a4a48acb4c8c079b2018474740844cf2b52","observation_id":"10514c65-8456-4621-9750-548c46acfb89","resolution":{"observed_at":"2026-08-12T16:38:43.673850Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.464715Z","title":"A study of the uniqueness of source code,","venue":null,"work_id":"d80f8d93-ed35-48ae-a0bd-744e01f98699","year":2010},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.811382Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:a25dbcc124a02e90389bc54f4fde55d4e577339cad888057d0eee2c1be5fb686","observation_id":"51d34f47-928c-4ef0-b6c9-2fea25c1c71e","resolution":{"observed_at":"2026-08-12T16:38:43.468959Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2312.15698","last_updated":"2025-09-03T01:34:48Z","snapshot_observed_at":"2026-08-14T02:03:48.111216Z","submitted_at":"2023-12-25T11:39:46Z","title":"RepairLLaMA: Efficient Representations and Fine-Tuned Adapters for Program Repair","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.15698","snapshot_observed_at":"2026-08-12T16:38:42.799148Z","title":"Available: https://doi.org/10.48550/arXiv.2312.15698","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.799148Z"},"links":{"cited_paper":"/paper/2312.15698","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:cedb08fef5375c0567e002abd8e2419de3f55c5dee601ea98ac111743380db5f","observation_id":"59e1695b-2a69-4bf8-ab28-b7dafbd097c9","resolution":{"observed_at":"2026-08-12T16:38:42.799148Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.450065Z","title":"Memorization without overfitting: Analyzing the training dynamics of large language models,","venue":null,"work_id":"0d765e7a-192a-4ac1-a141-3d9ab3e22db2","year":2022},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.819025Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:b4d7cff56751a61e8e77482f38187d89e34ae988116c5bf156f0e7139438e046","observation_id":"49ac9234-caa2-4cd3-aa83-76a8f17f36c4","resolution":{"observed_at":"2026-08-12T16:38:43.455722Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.492256Z","title":"The stack: 3 TB of permissively licensed source code,","venue":null,"work_id":"3aa96bc3-6c04-4443-afe7-2b2e2d0a2917","year":2023},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.807962Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:7371e52cf5ff2552f5398fca4d0f8a8045fab0bf0bdb8e84f483e38681a7f92f","observation_id":"41c367a4-fe6c-4c5f-af46-ae0e0176eb1d","resolution":{"observed_at":"2026-08-12T16:38:43.575402Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.435696Z","title":"Measuring massive multitask language understanding,","venue":null,"work_id":"4c3303b7-a062-4f17-a2d5-2bd24ce4ed38","year":2021},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.826381Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:c674d8e7a991d215be6400563db4def4fc69e00283e642f8f817066584429eb3","observation_id":"5b037e62-a0b0-4319-9405-f770df87ecad","resolution":{"observed_at":"2026-08-12T16:38:43.440137Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2308.08747","last_updated":"2025-01-05T04:09:00Z","snapshot_observed_at":"2026-08-10T01:52:30.559515Z","submitted_at":"2023-08-17T02:53:23Z","title":"An Empirical Study of Catastrophic Forgetting in Large Language Models During Continual Fine-tuning","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.08747","snapshot_observed_at":"2026-08-12T16:38:42.814911Z","title":"An empirical study of catastrophic forgetting in large language models during continual fine-tuning,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.814911Z"},"links":{"cited_paper":"/paper/2308.08747","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:3ab0568c3f51f233cb11fef0db9910ebb5622298fbaf6acfe3bff71c5ab13cab","observation_id":"aaaadc1c-7b6a-4bde-b9b6-f7a4fdaaff5e","resolution":{"observed_at":"2026-08-12T16:38:42.814911Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2108.07732","last_updated":"2021-08-16T03:57:30Z","snapshot_observed_at":"2026-08-02T19:23:53.535075Z","submitted_at":"2021-08-16T03:57:30Z","title":"Program Synthesis with Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.07732","snapshot_observed_at":"2026-08-12T16:38:42.838268Z","title":"Program synthesis with large language models,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.838268Z"},"links":{"cited_paper":"/paper/2108.07732","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:8e7b5346059a2536d6e2dd6b25b6aa0b3097b2a9f50423b188bc09acebdad6df","observation_id":"ccc78991-03b9-4c7d-9762-de97282f6fd7","resolution":{"observed_at":"2026-08-12T16:38:42.838268Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:42.822872Z","title":"Squad: 100, 000+ questions for machine comprehension of text,","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.822872Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:e1e89eedac2cfecdfc0b5949ecee02d88e4981288b354ff5efac5b664c63bbc4","observation_id":"a1acba78-afc4-4fa8-8b08-09bdf13a6c33","resolution":{"observed_at":"2026-08-12T16:38:42.822872Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2304.00385","last_updated":"2024-12-09T20:45:33Z","snapshot_observed_at":"2026-08-13T12:11:43.265410Z","submitted_at":"2023-04-01T20:57:33Z","title":"Keep the Conversation Going: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2304.00385","snapshot_observed_at":"2026-08-12T16:38:42.845327Z","title":"Keep the conversation going: Fixing 162 out of 337 bugs for $0.42 each using chatgpt,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.845327Z"},"links":{"cited_paper":"/paper/2304.00385","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:c4da60cd345015d8234bdcfc4f07b8075347bdefa4e412c46c031d2cc1490cfc","observation_id":"7d5cf801-2987-42b0-9aff-464a161cbd97","resolution":{"observed_at":"2026-08-12T16:38:42.845327Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:42.830570Z","title":"Evaluating large language models trained on code,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.830570Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:1e559fe4852aface20a83105c2e29265121701a87b4f341a639351103e2abc3b","observation_id":"a3a57390-4812-4472-819f-e5750cd47e0a","resolution":{"observed_at":"2026-08-12T16:38:42.830570Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.07974","last_updated":"2024-06-06T17:41:21Z","snapshot_observed_at":"2026-08-13T09:04:30.125777Z","submitted_at":"2024-03-12T17:58:04Z","title":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.07974","snapshot_observed_at":"2026-08-12T16:38:42.893040Z","title":"Livecodebench: Holistic and contamination free evalua- tion of large language models for code,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.893040Z"},"links":{"cited_paper":"/paper/2403.07974","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:b4b708711e948e9dca9646c425798df57a257dfd563fd1bb44a620d4207d0831","observation_id":"b293650d-04a7-415b-9989-f12de4f16fe7","resolution":{"observed_at":"2026-08-12T16:38:42.893040Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.303483Z","title":"Why does your data leak? uncovering the data leakage in cloud from mobile apps,","venue":null,"work_id":"4c908478-e445-43b2-9131-876a65c4b48c","year":2019},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.962606Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:282c4ce4cf0e44f79a833b4e855c2675c4ac8e52a2ced08f4fa592f21f1561a8","observation_id":"61797f08-cfb1-4f59-b0fd-409ea0d30555","resolution":{"observed_at":"2026-08-12T16:38:43.353535Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.414093Z","title":"Measuring coding challenge competence with APPS,","venue":null,"work_id":"ad9e8876-2fea-4113-af05-a5e05b0d1bbb","year":2021},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.841580Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:864be5464d3a4a71499166101d296ae1022c789d4e30a2fad862d9ae8e18c6e4","observation_id":"47de63b2-7354-4d2d-944e-93e212e32bf4","resolution":{"observed_at":"2026-08-12T16:38:43.418581Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T16:38:43.400852Z","title":"Detecting pretraining data from large language models,","venue":null,"work_id":"23cefabc-4f39-4e6a-a3e2-80345c120da4","year":2024},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.849344Z"},"links":{"citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:765146877067d836614765bc62f6ddd78963141b2e28e66ba3f4dfd9d6dcf187","observation_id":"aa29b1cc-1416-4b5b-b1ef-8677991695f3","resolution":{"observed_at":"2026-08-12T16:38:43.405143Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.07974","last_updated":"2024-06-06T17:41:21Z","snapshot_observed_at":"2026-08-13T09:04:30.125777Z","submitted_at":"2024-03-12T17:58:04Z","title":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.07974","snapshot_observed_at":"2026-08-12T16:38:42.958319Z","title":"Available: https://doi.org/10.48550/arXiv.2403.07974","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.958319Z"},"links":{"cited_paper":"/paper/2403.07974","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:07a90573e5d5fbe6bf8c846a04e8350c1a76fe3ea88ab9d4c8ae36a00915e9ec","observation_id":"3b338049-eaca-4c1a-bed3-2f8252848170","resolution":{"observed_at":"2026-08-12T16:38:42.958319Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T11:58:24.516369Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-12T16:38:42.833805Z","title":"Available: https://arxiv.org/abs/2107.03374","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.833805Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:e3c86d26a6cb6985b4ca2bf8a423ceac80b9d7e4e8f6577ba8eebd0363972895","observation_id":"371f832b-16fd-45fa-8db1-91e2625b18a2","resolution":{"observed_at":"2026-08-12T16:38:42.833805Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.10677","last_updated":"2023-09-27T01:15:49Z","snapshot_observed_at":"2026-08-13T10:09:54.548080Z","submitted_at":"2023-09-19T15:02:58Z","title":"Estimating Contamination via Perplexity: Quantifying Memorisation in Language Model Evaluation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.10677","snapshot_observed_at":"2026-08-12T16:38:42.292215Z","title":"Available: https://doi.org/10.48550/arXiv.2309.10677","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.292215Z"},"links":{"cited_paper":"/paper/2309.10677","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:cb4bdd94d36feb36ee50efabf4a06516ac3210e163a24f338f9aea1d4ab26b5e","observation_id":"860f927a-9d1b-4c35-b4fc-9d5cea66c2cc","resolution":{"observed_at":"2026-08-12T16:38:42.292215Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.11409","last_updated":"2024-06-19T02:37:50Z","snapshot_observed_at":"2026-08-12T23:41:14.320216Z","submitted_at":"2024-06-17T10:54:35Z","title":"CodeGemma: Open Code Models Based on Gemma","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.11409","snapshot_observed_at":"2026-08-12T16:38:42.656619Z","title":"Available: https://doi.org/10.48550/arXiv.2406.11409","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?","version":3},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-12T16:38:42.656619Z"},"links":{"cited_paper":"/paper/2406.11409","citing_paper":"/paper/2411.13323"},"observation_digest":"sha256:0279965c6118ad0222700bfc0050a01403e8dec03a8ada552219b71d4f910940","observation_id":"df50b642-a106-4b59-b337-7ca5e5baebda","resolution":{"observed_at":"2026-08-12T16:38:42.656619Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2411.13323","last_updated":"2025-03-31T13:02:51Z","latest_version":3,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-15T07:29:39.426041Z","submitted_at":"2024-11-20T13:46:04Z","title":"Are Large Language Models Memorizing Bug Benchmarks?"},"reference_resolution":{"displayed":45,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":23,"verified_exact":0,"verified_fuzzy":22},"total_outbound_references":45},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"thesis":"As of 15 August 2026, this Paper Citation Record lists 45 of 45 outbound references and 1 inbound Pith citation observation for arXiv:2411.13323."}