{"as_of":"2026-08-09T17:24:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:d47541360dfd9256b99f6e8c2071299ebb91e2c2c0e482fd6e1d05c0d25a9577","coverage":[{"denominator":97,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":97,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-02T03:41:08.536430Z","state":"measured"},{"denominator":98,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":98,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-09T06:31:02.800959+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-04T01:05:44.021875Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2607.13820","snapshot_observed_at":"2026-08-04T01:05:44.021875Z","title":"Probe: Benchmarking code generation in large language models,","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2608.00661","last_updated":"2026-08-01T13:31:42Z","snapshot_observed_at":"2026-08-08T03:15:07.866707Z","submitted_at":"2026-08-01T13:31:42Z","title":"Unreliable in Practice? A Comprehensive Study of Errors in LLM-Generated Code","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-04T01:05:44.021875Z"},"links":{"cited_paper":"/paper/2607.13820","citing_paper":"/paper/2608.00661"},"observation_digest":"sha256:5d5e182e2abd345a658ebb6a6fa389a4f5a5f0dbeaaf7937b992bbb2208c769a","observation_id":"0f48bb42-a000-4a5c-a932-9d11f02dcaab","resolution":{"observed_at":"2026-08-04T01:05:44.021875Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2607.13820/citation-record","integrity":"/paper/2607.13820/integrity","json":"/paper/2607.13820/citation-record.json","paper":"/paper/2607.13820"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:40:59.041633Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-02T03:40:59.041633Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:e7669f1268ed063ed60cfb88294e688c0aa2aecd2e2f817a932134caa980e8d4","observation_id":"a410a632-1763-4eae-b526-daed9a6618b6","resolution":{"observed_at":"2026-08-02T03:40:59.041633Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:40:59.077840Z","title":", title =","venue":null,"work_id":null,"year":1971},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-02T03:40:59.077840Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:99dba05d4e6b002daa8ba27f327bc3afda7526c6b27dfc7e9b30d7fbc5c9abfd","observation_id":"46f6024f-dc71-42bb-9e7d-8536f2e661b7","resolution":{"observed_at":"2026-08-02T03:40:59.077840Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:40:59.127050Z","title":", title =","venue":null,"work_id":null,"year":1977},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-02T03:40:59.127050Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:c238fce79455aef1e9778db4e97008fa910402e5d16fc0f9dd97562c133049b6","observation_id":"5555c998-53eb-458d-a936-7eccee424179","resolution":{"observed_at":"2026-08-02T03:40:59.127050Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:40:59.187776Z","title":null,"venue":null,"work_id":null,"year":1957},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-02T03:40:59.187776Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:b68add1dba01d44fea5ccc887c4f286bd6c6180eef4d8cc3ff1416af239b8fd4","observation_id":"04b2ea7e-5742-4ea3-8061-20c2378e1832","resolution":{"observed_at":"2026-08-02T03:40:59.187776Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:40:59.240390Z","title":"2021 , eprint=","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-02T03:40:59.240390Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:3fc9cf69ad4a24a0e256edf038fe91fa302cd4850e05aefba14f5b57b34aff2b","observation_id":"ca44938c-c300-45e0-baed-3f7865d2123a","resolution":{"observed_at":"2026-08-02T03:40:59.240390Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:40:59.328176Z","title":"Alice and Rice, Andrew and Rifkin, Devon and Simister, Shawn and Sittampalam, Ganesh and Aftandilian, Edward , title =","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-02T03:40:59.328176Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:96a0381b09130fdad4f1f3b95138bd09d5538440394d90769ff275d980e5a469","observation_id":"db678837-a4c4-4847-947b-1c4be834457f","resolution":{"observed_at":"2026-08-02T03:40:59.328176Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:40:59.428297Z","title":"2024 , eprint=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-02T03:40:59.428297Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:6954a94d1fd93f73b905f93bc15e30ca826bf1246915428f55dfb12b5492cc0b","observation_id":"b8194cc0-d9d1-4ecd-8966-eb885e5e2480","resolution":{"observed_at":"2026-08-02T03:40:59.428297Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.15115","last_updated":"2025-01-03T02:18:21Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-12-19T17:56:09Z","title":"Qwen2.5 Technical Report","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.15115","snapshot_observed_at":"2026-08-02T03:40:59.516100Z","title":"5 Technical Report , author=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-02T03:40:59.516100Z"},"links":{"cited_paper":"/paper/2412.15115","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:5894e089d1ff42a006dda1da4ae4f0b8ae3719d7684013bd2778cf1dc490dfb9","observation_id":"481876de-ea26-4800-8cc8-012518bb2298","resolution":{"observed_at":"2026-08-02T03:40:59.516100Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:40:59.591302Z","title":"2024 , publisher=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-02T03:40:59.591302Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:a207d7a293ea6c3cf6ebf462ef8d3cb60adffd4055f7284cb54e3b58845d69ed","observation_id":"56a8bc06-989c-4ddc-b604-4033f04747ee","resolution":{"observed_at":"2026-08-02T03:40:59.591302Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:40:59.681179Z","title":"Language Models are Few-Shot Learners , url =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-02T03:40:59.681179Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:2e84aeefdd5619c5b5fe8099fa4c056411f0cf0cf949effafe078a3922361499","observation_id":"bc7dcfd6-2643-4eed-b818-f124e6e1eb63","resolution":{"observed_at":"2026-08-02T03:40:59.681179Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:40:59.764700Z","title":"2021 , eprint=","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-02T03:40:59.764700Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:61257c3cf690a88fec580cd5789136b859b4a99314e099db38e23e331f2d01a3","observation_id":"b1d2a032-cfc3-4f04-a76a-bc1b15b2d774","resolution":{"observed_at":"2026-08-02T03:40:59.764700Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:40:59.836997Z","title":"2021 , eprint=","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-02T03:40:59.836997Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:7d79ac4eb42be88ddf85fc501cf49dbd62e25480789f812f106270cb94876f6e","observation_id":"34e701d1-0d7c-482a-a9e8-58ae65754832","resolution":{"observed_at":"2026-08-02T03:40:59.836997Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:40:59.912370Z","title":"2023 , eprint=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-02T03:40:59.912370Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:26465d27f9ccb5e503ff0e9e07c4944b88a016b170be6550f3d90f37c79be2fd","observation_id":"737ed91b-eb48-4f57-820f-35a99cf0d527","resolution":{"observed_at":"2026-08-02T03:40:59.912370Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:40:59.994552Z","title":"2021 , eprint=","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-02T03:40:59.994552Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:7d393bf023d2cbaf13c160eeb61ac830ad06e262838862a914ed1388137e871d","observation_id":"c286d5d3-bd91-4dcf-ba30-de49db1c824d","resolution":{"observed_at":"2026-08-02T03:40:59.994552Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:00.032812Z","title":"Attention is All you Need , url =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:00.032812Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:f65f304db5af2391498a13676d2141f6bac76da0cfc66302b8f56778056d77d0","observation_id":"ab136e6a-125f-46b0-86a6-c37ca5c5311f","resolution":{"observed_at":"2026-08-02T03:41:00.032812Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1007/978-3-642-24797-2_4","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Long Short-Term Memory","venue":"Studies in computational intelligence","work_id":"cf4dfa3c-c3f9-4053-9591-f3c17b6f2f6e","year":2012},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:00.083856Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:7468ed1fcdb3d4e471f850fd99e8ee61ece9c50e8e15d2e06acb72610c25620e","observation_id":"0f4253f5-7891-434b-b9fe-eea90dcd8fdc","resolution":{"observed_at":"2026-08-02T03:43:22.735297Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:00.142904Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:00.142904Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:e1ca3f31ee892f4364be259d368ef4f4c4f48306ae5a228e7d3c136d9fb488c5","observation_id":"e8829be0-b76f-406a-811d-5261067a277c","resolution":{"observed_at":"2026-08-02T03:41:00.142904Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:00.197477Z","title":"2024 , eprint=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:00.197477Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:d8622b8a73e7ef76cbe2eac498f5b6be2c0a6bb4d055b92cb7969403f3cb537a","observation_id":"5f7b12dd-6f89-4f12-9f6d-4a0a5ac213ac","resolution":{"observed_at":"2026-08-02T03:41:00.197477Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:00.254344Z","title":"2002 , publisher =","venue":null,"work_id":null,"year":2002},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:00.254344Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:0813ea2f7b5acd819dcc1601c7bcda5d5e13e00b22ee408c89775938c01a9333","observation_id":"4b13adc5-9e79-44a0-a88e-5032b32c363e","resolution":{"observed_at":"2026-08-02T03:41:00.254344Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:00.340873Z","title":"Text summarization branches out , pages=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:00.340873Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:542a41fa768363df24f78b1e85090dfed74bad4b9d354598a29895c267eadaee","observation_id":"cef076cf-bbeb-417f-a1df-92be8bb7f835","resolution":{"observed_at":"2026-08-02T03:41:00.340873Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:00.403280Z","title":"Proceedings of the acl workshop on intrinsic and extrinsic evaluation measures for machine translation and/or summarization , pages=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:00.403280Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:10f5a9c78f4c36b060e5dacb740cdb2ccec46c808d62bb855dff0e95c1e9ae14","observation_id":"2543796d-21fe-43a7-bb3e-8c34101e8b3c","resolution":{"observed_at":"2026-08-02T03:41:00.403280Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:00.462363Z","title":"2020 , eprint=","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:00.462363Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:82c540d0539945714d059c5d443f82325aacea110f0677cccbcfb8e36bd0af48","observation_id":"527405ba-9a91-436e-846c-d94289d7ef8d","resolution":{"observed_at":"2026-08-02T03:41:00.462363Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:00.535796Z","title":"1992 , publisher=","venue":null,"work_id":null,"year":1992},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:00.535796Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:88895ac60908179caa54b02898cd77654ce4900077d5fd83adc94ce82793b9e8","observation_id":"2bc85aca-64f6-414d-86b1-2419ec2fc1d0","resolution":{"observed_at":"2026-08-02T03:41:00.535796Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:00.598390Z","title":"Synthesize, Execute and Debug: Learning to Repair for Neural Program Synthesis , url =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:00.598390Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:02f165ad24f29a39f09d0bd99895bca15a6e454cc30f2d8ba7ac99a093847f9b","observation_id":"11f245cf-2c48-476d-a9ad-d23f72615d46","resolution":{"observed_at":"2026-08-02T03:41:00.598390Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:00.682028Z","title":"2024 , eprint=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:00.682028Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:572fa28f87e243ac1f328b08f27f9e79255ab5e35567bd24479cea862f73685f","observation_id":"9ba4e8c5-9750-471e-b948-7261049bf8cc","resolution":{"observed_at":"2026-08-02T03:41:00.682028Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:00.751011Z","title":"2023 , eprint=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:00.751011Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:0fe5c09e75e056e51530d033725a8fbd4ec803f76ec991510230abf344300edf","observation_id":"df89dfc2-c769-4ba0-9626-0a7a2e75aaa0","resolution":{"observed_at":"2026-08-02T03:41:00.751011Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:00.833125Z","title":"2024 , eprint=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:00.833125Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:1288377e172e48fad05b887742d3380ca7b1375909549ea18aeaaa787051c776","observation_id":"957690fe-b84f-4000-ad60-393ca88e9371","resolution":{"observed_at":"2026-08-02T03:41:00.833125Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:00.896613Z","title":"2017 , volume =","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:00.896613Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:a4ac65f363ef56bbf78e533c62df74d20ae4d327f5e833a2edfe7f834cfef7f5","observation_id":"bb6c933d-b19c-42cb-bdfa-d60bca05576b","resolution":{"observed_at":"2026-08-02T03:41:00.896613Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:00.949870Z","title":"Predicting a Correct Program in Programming by Example","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:00.949870Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:3c292de0f3ae9165af0070a9fb9e6fccc841fad4276d59fe33a8d1a4ccc7229f","observation_id":"7a3fc547-5788-410d-a981-6dfcc876fe09","resolution":{"observed_at":"2026-08-02T03:41:00.949870Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.006976Z","title":"SPoC: Search-based Pseudocode to Code , url =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.006976Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:e308227bd1682de2ad9893cd9fe6bc082989d6cf01b1a35fbc11ec2ac15feafa","observation_id":"d0cfd897-0205-4523-8c02-61ed823bd4c1","resolution":{"observed_at":"2026-08-02T03:41:01.006976Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.101322Z","title":"Neural Attribution for Semantic Bug-Localization in Student Programs , url =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.101322Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:9f9746da5608a47fae761e42ac9517533988a68e582977acba6839bd2028bad7","observation_id":"4f7c1253-f23b-4c76-8a62-dad4aa3e61ae","resolution":{"observed_at":"2026-08-02T03:41:01.101322Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.162778Z","title":"2021 , eprint=","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.162778Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:e909e04413e28cb8316d0fb808d485da71b01662f0207e04a4c1b1a46a0b733c","observation_id":"e7f33a20-97cd-4cd7-b250-1b049ce514a2","resolution":{"observed_at":"2026-08-02T03:41:01.162778Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.232166Z","title":"Liu , title =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.232166Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:9f886dbd743d1062be06bdb706efac2d02bcdf1f81a17d135f20ebfe8dddb545","observation_id":"b57e87ba-a056-40b5-86b1-beeba0e1b69b","resolution":{"observed_at":"2026-08-02T03:41:01.232166Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.286929Z","title":"Distributed Representations of Words and Phrases and their Compositionality , url =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.286929Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:c8eb540d22421e201e44ad85b97ec774b65197ed578852e7c7dabfb46fd8a8cb","observation_id":"040faf8b-a86b-492f-9022-c124e6de4d4f","resolution":{"observed_at":"2026-08-02T03:41:01.286929Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.377487Z","title":"Proceedings of the 2014 conference on empirical methods in natural language processing (EMNLP) , pages=","venue":null,"work_id":null,"year":2014},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.377487Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:f69eb4e7064fc83f42e537653ac0e1b5b044467867c55385b2dab01964fe4e9e","observation_id":"b2ec3db1-d854-47de-9cd7-6919cdd8b47d","resolution":{"observed_at":"2026-08-02T03:41:01.377487Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.424056Z","title":"and Fei-Fei, Li , title =","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.424056Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:8c9a589d0b6fcd913f3ff070d7aad8dd559dd22b665678f8b481641834f0e4a1","observation_id":"493145d8-bd80-4e8e-8ae1-32cbcfaf7242","resolution":{"observed_at":"2026-08-02T03:41:01.424056Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.474006Z","title":"2024 , eprint=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.474006Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:ef7cf069c6ae7ab89c7354ba23fe8bdb662ab6146128043d6a6ae4ef2e915a0b","observation_id":"31fc5312-2cc7-4b43-9667-843c3dc6e76f","resolution":{"observed_at":"2026-08-02T03:41:01.474006Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.522051Z","title":"2019 , eprint=","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.522051Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:e052e244e92282c4ab3023348b024129b216f7b4d25e79c2881cab761e4f08a3","observation_id":"daf10398-a8ed-44b7-a6de-9a31f26315b4","resolution":{"observed_at":"2026-08-02T03:41:01.522051Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.585939Z","title":"2021 , eprint=","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.585939Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:94075ab4b36d0a45ac5ad17ca9f278b2df6ed66931f9abc28bf08d96a908cd09","observation_id":"563c8154-874e-47b5-ad18-dd3fec4f8107","resolution":{"observed_at":"2026-08-02T03:41:01.585939Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.639757Z","title":"OpenAI blog , volume=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":40,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.639757Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:429dc31f4bf749f933b52685b854df956fb717b4bfff5fde5c2d5215bc78940c","observation_id":"2468b25b-588c-4326-a3b9-8032da3bf6f4","resolution":{"observed_at":"2026-08-02T03:41:01.639757Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.675781Z","title":"2023 , eprint=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":41,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.675781Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:7915ab01dbba33fa299d7bc90b841946d9f278d7ebd339e1b4081c333fa73061","observation_id":"652c8d91-ed11-4f33-85fa-d18e4c36dfdf","resolution":{"observed_at":"2026-08-02T03:41:01.675781Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.718188Z","title":"2021 , eprint=","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":42,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.718188Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:a3b4788738718051a8696f22e45631b04e98d67011a95b769494cbff52831f6c","observation_id":"932c8d21-35d5-41ac-bac5-cb0944d16ccc","resolution":{"observed_at":"2026-08-02T03:41:01.718188Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.769383Z","title":"Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing , pages=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":43,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.769383Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:bb1e8ea41b1c98150523843347ee447b99d0bf969faa5cb78769c6e867f9cb22","observation_id":"a2f7f0e5-ae73-4fd4-b1a3-90c92128c65f","resolution":{"observed_at":"2026-08-02T03:41:01.769383Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.803262Z","title":"2021 , eprint=","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":44,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.803262Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:1eacd08b885b08422b84215c9505d7d3d0a790babbf84112e4d8b802dd275e0a","observation_id":"e6940600-7ed7-43d8-bca9-0c5d656ae65d","resolution":{"observed_at":"2026-08-02T03:41:01.803262Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.871602Z","title":"2022 , eprint=","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":45,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.871602Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:4da97572cc6081764d6ee3f6a4c428ffead37a28ac9eed610ebb11ecd59d5808","observation_id":"dc5f6d40-fcae-4e47-b8ef-bf8447568779","resolution":{"observed_at":"2026-08-02T03:41:01.871602Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.924993Z","title":", title =","venue":null,"work_id":null,"year":1980},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":46,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.924993Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:bc22bb678e6c934ab77d126e4b72e1616dfd57f2ecffeb3e93a08eb37bdbf6a8","observation_id":"50612aee-d8d4-49cf-a4f6-b19898bee443","resolution":{"observed_at":"2026-08-02T03:41:01.924993Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:01.977621Z","title":"2023 , eprint=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":47,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:01.977621Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:948bc7987c19cf95480a5bcf1a784f71f2bacce83f901735c2e5ab4190697f5a","observation_id":"96776520-2829-4aac-8e21-a59e9828117e","resolution":{"observed_at":"2026-08-02T03:41:01.977621Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:02.029969Z","title":"2023 , eprint=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":48,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:02.029969Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:12b10fa5d36ba701c7e68713e6f07085311a4bce6eaf1b21267f6e8b83a8f2bb","observation_id":"fcfae5a5-ebf1-4e7b-ad33-06f6bb59bbfc","resolution":{"observed_at":"2026-08-02T03:41:02.029969Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1902.01007","last_updated":"2019-06-24T16:02:01Z","snapshot_observed_at":"2026-07-06T07:30:52.835880Z","submitted_at":"2019-02-04T01:54:19Z","title":"Right for the Wrong Reasons: Diagnosing Syntactic Heuristics in Natural Language Inference","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1902.01007","snapshot_observed_at":"2026-08-02T03:41:02.119264Z","title":"arXiv preprint arXiv:1902.01007 , year=","venue":null,"work_id":null,"year":1902},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":49,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:02.119264Z"},"links":{"cited_paper":"/paper/1902.01007","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:6bf5bbeec865d653c47fc16feff704a7aac817750917412910b3757808841f97","observation_id":"d0708426-74c3-442a-bb29-d852c03f1b56","resolution":{"observed_at":"2026-08-02T03:41:02.119264Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:02.196404Z","title":"2019 , eprint=","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":50,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:02.196404Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:bbe45355a891468b35441917404427d1bb966220c0ca59b89d20d0ce007a9522","observation_id":"c50e8c9f-9f79-4861-8750-e87aca657167","resolution":{"observed_at":"2026-08-02T03:41:02.196404Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:02.276767Z","title":"2022 , eprint=","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":51,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:02.276767Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:363044ec1b3f5776a2bcc4ab470bdbc5a9ea081785f59ca68ca92a2a183fef34","observation_id":"00153917-8540-4ace-ad5b-d0ab58743f6b","resolution":{"observed_at":"2026-08-02T03:41:02.276767Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:02.328305Z","title":"2023 , eprint=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":52,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:02.328305Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:36b8027b37b66d6d247f69080702f12949b6ad81e2401e2301be6700aa572e8e","observation_id":"8cdbf6bf-c228-48bb-9068-74f0e2fb4e12","resolution":{"observed_at":"2026-08-02T03:41:02.328305Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:02.435612Z","title":"2020 , eprint=","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":53,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:02.435612Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:8e74cab553f8a2e2d5694419766c369b0258052a41597dcaad5ab085fc0f47c0","observation_id":"77bc4aa6-e016-447c-9548-7885e188a949","resolution":{"observed_at":"2026-08-02T03:41:02.435612Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:02.486791Z","title":"2023 , eprint=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":54,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:02.486791Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:4aeda82bb95e0ef2da62a246bd474623bdd3acad51e8ce0d610015b920b18736","observation_id":"f4eab490-835d-4dcf-a8ae-814740c7fdc1","resolution":{"observed_at":"2026-08-02T03:41:02.486791Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:02.489711Z","title":"2024 , eprint=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":55,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:02.489711Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:7917da4549f17dc040f40b70e86b757b34a450fe02bf0c6f7b49a952d92cf177","observation_id":"1421c880-32d1-474a-ac36-7f6084a89b1d","resolution":{"observed_at":"2026-08-02T03:41:02.489711Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:02.555293Z","title":"Training language models to follow instructions with human feedback , url =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":56,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:02.555293Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:538bee726ec34bf324c90474c7905f656a47d11991558ff1f5544b8d5fa9a90c","observation_id":"49c2f4be-60da-4c7b-87cd-8254a55b813b","resolution":{"observed_at":"2026-08-02T03:41:02.555293Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:02.693353Z","title":"2024 , eprint=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":57,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:02.693353Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:3f0df92822861cdfac8218d69157518bce0506049fd96af3ba76d3976ab1a707","observation_id":"bd143f87-cee7-4d53-b105-b3510a51e39d","resolution":{"observed_at":"2026-08-02T03:41:02.693353Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:02.855468Z","title":"2023 , eprint=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":58,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:02.855468Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:a86d148dc5b229a7124bf3a397228dd38cf92813fca603ba7d77075b331ab588","observation_id":"0cdc6d04-4602-4cbe-9183-6aa0d4f67aba","resolution":{"observed_at":"2026-08-02T03:41:02.855468Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.11747","last_updated":"2023-10-23T01:49:32Z","snapshot_observed_at":"2026-08-07T22:56:15.588770Z","submitted_at":"2023-05-19T15:36:27Z","title":"HaluEval: A Large-Scale Hallucination Evaluation Benchmark for Large Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.11747","snapshot_observed_at":"2026-08-02T03:41:02.990486Z","title":"arXiv preprint arXiv:2305.11747 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":59,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:02.990486Z"},"links":{"cited_paper":"/paper/2305.11747","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:7a29a2dfa09b580302fef2c2548c4d8021a79f2658c589160e018746e83962e2","observation_id":"fc2a8aa5-5343-4316-9cc9-3159358ed8aa","resolution":{"observed_at":"2026-08-02T03:41:02.990486Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:03.113431Z","title":"Advances in neural information processing systems , volume=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":60,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:03.113431Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:d12756b43c8c710c9a692eefd79f72981ce7d3e57df702f724738d014678d41d","observation_id":"0cedbebf-9d83-4555-9b52-d2d936a808d5","resolution":{"observed_at":"2026-08-02T03:41:03.113431Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:03.187782Z","title":"Proceedings 2003 VLDB Conference , pages=","venue":null,"work_id":null,"year":2003},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":61,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:03.187782Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:c3dfb9b9cbf7b92e0e07358e844f4982878364b90f0824f4162bfdc2c783e5ee","observation_id":"e145c4ee-625c-401d-8199-49cc5785808a","resolution":{"observed_at":"2026-08-02T03:41:03.187782Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:03.310408Z","title":"2021 , publisher =","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":62,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:03.310408Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:891d84fa875b2040f4439bfb92003cc694c31d355afef28c8a5b18f718febb80","observation_id":"f10755b6-7956-4da4-9cfb-391268150286","resolution":{"observed_at":"2026-08-02T03:41:03.310408Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:03.430829Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":63,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:03.430829Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:a772191fc7d1947803bc74e9c4254702dd58ca3218456666a585a50c8065673a","observation_id":"df0dfbb5-133d-40d1-8560-6176b9010b83","resolution":{"observed_at":"2026-08-02T03:41:03.430829Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1808.09588","last_updated":"2018-08-29T00:08:25Z","snapshot_observed_at":"2026-08-08T17:13:58.133122Z","submitted_at":"2018-08-29T00:08:25Z","title":"Mapping Language to Code in Programmatic Context","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1808.09588","snapshot_observed_at":"2026-08-02T03:41:03.553027Z","title":"arXiv preprint arXiv:1808.09588 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":64,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:03.553027Z"},"links":{"cited_paper":"/paper/1808.09588","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:5f714cc192a4f5735e484d9dbc000f327b42c3a9a970e50b3a0010adc2564d97","observation_id":"7a7ba2e0-ce04-4245-875e-29e24b32a7e0","resolution":{"observed_at":"2026-08-02T03:41:03.553027Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:03.670850Z","title":"2024 , eprint=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":65,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:03.670850Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:1ce0a49a464969b9a2bc5c23801c72260c72d3c35c82e21ab185b024dbcbb096","observation_id":"e9b2af0c-f4eb-4659-8862-7a2d63f61952","resolution":{"observed_at":"2026-08-02T03:41:03.670850Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:03.792906Z","title":"2022 , eprint=","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":66,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:03.792906Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:a0843c44dcb0d01ba30d36f1cdab126c75beb828c730f1b8ef3dc87c1ed6b187","observation_id":"1d9daa2e-6d8a-459d-b00e-f3f679847326","resolution":{"observed_at":"2026-08-02T03:41:03.792906Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:03.873151Z","title":"2022 , eprint=","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":67,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:03.873151Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:b887ad58ac88380d50ee8be56234bf1377b8b9fba1e6844e4056bc80555843a7","observation_id":"485e5059-decb-40fe-9846-dd5874deaa3b","resolution":{"observed_at":"2026-08-02T03:41:03.873151Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:03.997658Z","title":"2024 , eprint=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":68,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:03.997658Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:c094b93f81e81850824d18faec8c624cc18006ba91d7e6e15413140f377ae9ef","observation_id":"c0bba666-ab21-4e44-866a-0539991f3d35","resolution":{"observed_at":"2026-08-02T03:41:03.997658Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:04.116893Z","title":"2024 , eprint=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":69,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:04.116893Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:ce7e93d8c4fcf5086427ea6b6c0146ea3c0c5077628df68fbcb8c0395b9fa595","observation_id":"a5989be4-e0d2-47cf-8527-6545042c4728","resolution":{"observed_at":"2026-08-02T03:41:04.116893Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:04.236798Z","title":"International conference on machine learning , pages=","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":70,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:04.236798Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:0364eacb1496a5f338062e43f3655346c282d91389355c45067186ef5abccb45","observation_id":"0ef7664c-32b3-40ae-9415-9e80a88179af","resolution":{"observed_at":"2026-08-02T03:41:04.236798Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2012.04584","last_updated":"2022-08-04T17:36:08Z","snapshot_observed_at":"2026-08-05T21:44:05.382799Z","submitted_at":"2020-12-08T17:36:34Z","title":"Distilling Knowledge from Reader to Retriever for Question Answering","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2012.04584","snapshot_observed_at":"2026-08-02T03:41:04.355532Z","title":"arXiv preprint arXiv:2012.04584 , year=","venue":null,"work_id":null,"year":2012},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":71,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:04.355532Z"},"links":{"cited_paper":"/paper/2012.04584","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:0c8d0e50d49fffebd714bfbaffbe675e7c19ec62cfc3a72c205d7957f630c87f","observation_id":"4370a0a0-4ad2-4be9-b247-33a7b29c51c6","resolution":{"observed_at":"2026-08-02T03:41:04.355532Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.20050","last_updated":"2023-05-31T17:24:00Z","snapshot_observed_at":"2026-08-05T13:11:04.104454Z","submitted_at":"2023-05-31T17:24:00Z","title":"Let's Verify Step by Step","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.20050","snapshot_observed_at":"2026-08-02T03:41:04.479139Z","title":"arXiv preprint arXiv:2305.20050 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":72,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:04.479139Z"},"links":{"cited_paper":"/paper/2305.20050","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:18a3bbb7a5b448d6aba20436e06a2c59b1465b474541c8802f92906283828cc5","observation_id":"46c049ac-8041-4a70-b333-fc67f9a7f2de","resolution":{"observed_at":"2026-08-02T03:41:04.479139Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2205.12443","last_updated":"2022-10-21T20:08:11Z","snapshot_observed_at":"2026-07-06T13:13:37.143547Z","submitted_at":"2022-05-25T02:22:30Z","title":"Generating Natural Language Proofs with Verifier-Guided Search","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2205.12443","snapshot_observed_at":"2026-08-02T03:41:04.562180Z","title":"arXiv preprint arXiv:2205.12443 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":73,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:04.562180Z"},"links":{"cited_paper":"/paper/2205.12443","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:a1a8ea39ea1c69bdb3490697743f5585bea5480356d3da6d1b281ab5b15cc8cb","observation_id":"39316138-77e0-4488-9ed2-8989be2131aa","resolution":{"observed_at":"2026-08-02T03:41:04.562180Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.06161","last_updated":"2023-12-13T14:44:10Z","snapshot_observed_at":"2026-07-06T15:25:35.930688Z","submitted_at":"2023-05-09T08:16:42Z","title":"StarCoder: may the source be with you!","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.06161","snapshot_observed_at":"2026-08-02T03:41:04.657260Z","title":"arXiv preprint arXiv:2305.06161 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":74,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:04.657260Z"},"links":{"cited_paper":"/paper/2305.06161","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:90bdad844780e23f0469b8c0daa1bb860d37aa94e3d4965cf2637d967af1ac8f","observation_id":"b91aafdb-bc61-4327-91df-217fb0944e1d","resolution":{"observed_at":"2026-08-02T03:41:04.657260Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1803.02324","last_updated":"2018-04-16T22:14:06Z","snapshot_observed_at":"2026-08-06T11:21:56.268167Z","submitted_at":"2018-03-06T18:23:08Z","title":"Annotation Artifacts in Natural Language Inference Data","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1803.02324","snapshot_observed_at":"2026-08-02T03:41:04.739473Z","title":"arXiv preprint arXiv:1803.02324 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":75,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:04.739473Z"},"links":{"cited_paper":"/paper/1803.02324","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:3793a7018afcc4780b73d3894fbb02582ed68b1b74426e6ec46e5a4cd25f3785","observation_id":"fba60d61-8335-44a9-becd-1728c132b945","resolution":{"observed_at":"2026-08-02T03:41:04.739473Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.04667","last_updated":"2025-04-02T15:17:02Z","snapshot_observed_at":"2026-07-06T18:58:35.218405Z","submitted_at":"2024-08-06T16:43:35Z","title":"Non-Determinism of \"Deterministic\" LLM Settings","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.04667","snapshot_observed_at":"2026-08-02T03:41:04.881119Z","title":"arXiv preprint arXiv:2408.04667 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":76,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:04.881119Z"},"links":{"cited_paper":"/paper/2408.04667","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:6063591fac64e6c8929571299d0c03bbfed69eba2fd7794c22d6a5f5fc440407","observation_id":"7e65b358-3a1f-4f0f-b8ae-b5d39fada2a7","resolution":{"observed_at":"2026-08-02T03:41:04.881119Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:05.042606Z","title":"Large Language Models are Zero-Shot Reasoners , url =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":77,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:05.042606Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:e078b5b55211412d2be17931472848b2baa854cd8f6308c142ef996c2288d93c","observation_id":"c1899611-320c-41f7-9c33-ebae04e3f7ce","resolution":{"observed_at":"2026-08-02T03:41:05.042606Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:05.143110Z","title":"Density-Based Clustering in Spatial Databases: The Algorithm GDBSCAN and Its Applications , journal =","venue":null,"work_id":null,"year":1998},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":78,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:05.143110Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:2a7023ef657126463d894aaee5cd2a3223d3b74faa2e21586229cef3587364e5","observation_id":"dbd2cbc5-ae6f-4ab3-8bab-0d48cb393f64","resolution":{"observed_at":"2026-08-02T03:41:05.143110Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.06204","last_updated":"2025-04-09T13:54:59Z","snapshot_observed_at":"2026-08-07T19:39:57.333135Z","submitted_at":"2024-06-26T16:34:33Z","title":"A Survey on Mixture of Experts in Large Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.06204","snapshot_observed_at":"2026-08-02T03:41:05.245748Z","title":"arXiv preprint arXiv:2407.06204 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":79,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:05.245748Z"},"links":{"cited_paper":"/paper/2407.06204","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:9eaf15dfdece6e80d943245018b99d19b79260ef47f4a7a08ada679a6e1e4e83","observation_id":"747fa17c-127d-4c4e-a779-9cd649416b2a","resolution":{"observed_at":"2026-08-02T03:41:05.245748Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1701.06538","last_updated":"2017-01-23T18:10:00Z","snapshot_observed_at":"2026-07-06T05:27:13.416519Z","submitted_at":"2017-01-23T18:10:00Z","title":"Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1701.06538","snapshot_observed_at":"2026-08-02T03:41:05.393633Z","title":"arXiv preprint arXiv:1701.06538 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":80,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:05.393633Z"},"links":{"cited_paper":"/paper/1701.06538","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:7033404d54ad4b26d20d643c991157127d47cdaf8571b9bbd2d1e745e3cc5edf","observation_id":"e168f7ef-749e-474a-b1b4-521536b62bc4","resolution":{"observed_at":"2026-08-02T03:41:05.393633Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:05.619368Z","title":"Journal of Machine Learning Research , volume=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":81,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:05.619368Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:af8221704c87c5ad1d7bb2a45c95ff40d733dde026a648663e2bc52354571ce3","observation_id":"4a630a48-8142-4033-9285-d8cd0de50098","resolution":{"observed_at":"2026-08-02T03:41:05.619368Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.19437","last_updated":"2025-02-18T17:26:38Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-12-27T04:03:16Z","title":"DeepSeek-V3 Technical Report","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.19437","snapshot_observed_at":"2026-08-02T03:41:05.769361Z","title":"arXiv preprint arXiv:2412.19437 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":82,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:05.769361Z"},"links":{"cited_paper":"/paper/2412.19437","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:20757a690270ec39cbfeba3a96cca5e2894bedf81d8442cce7f7048d1595ec24","observation_id":"42a60d18-10d9-41e5-b4f0-24799f274b3b","resolution":{"observed_at":"2026-08-02T03:41:05.769361Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.12948","last_updated":"2026-01-04T03:57:36Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-01-22T15:19:35Z","title":"DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.12948","snapshot_observed_at":"2026-08-02T03:41:05.979493Z","title":"arXiv preprint arXiv:2501.12948 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":83,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:05.979493Z"},"links":{"cited_paper":"/paper/2501.12948","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:a1a5c2db0b660c38703557ff8a4174d8169107cff10f5800993db63db1c3ef8e","observation_id":"769737aa-f3e4-413d-9705-1ceb5af6b6c1","resolution":{"observed_at":"2026-08-02T03:41:05.979493Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:06.143522Z","title":"Journal of medical Internet research , volume=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":84,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:06.143522Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:3048e0438321df51a743740bf57b01baca7e170cc60c31fc31b14cb744c6ff96","observation_id":"2358ba9e-178d-4abc-a77b-bc30fbaf5b3d","resolution":{"observed_at":"2026-08-02T03:41:06.143522Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2010.15980","last_updated":"2020-11-07T05:33:35Z","snapshot_observed_at":"2026-08-05T14:17:34.026911Z","submitted_at":"2020-10-29T22:54:00Z","title":"AutoPrompt: Eliciting Knowledge from Language Models with Automatically Generated Prompts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2010.15980","snapshot_observed_at":"2026-08-02T03:41:06.319054Z","title":"arXiv preprint arXiv:2010.15980 , year=","venue":null,"work_id":null,"year":2010},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":85,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:06.319054Z"},"links":{"cited_paper":"/paper/2010.15980","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:a9281b1a2cfa11a9d9bcb769c03a5a3ac3cbd08c012c53ce6d99fd8ac38b0bcd","observation_id":"b71dbb2c-c9f4-4dac-a76b-c9a7de4fbbf6","resolution":{"observed_at":"2026-08-02T03:41:06.319054Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.06608","last_updated":"2025-02-26T18:59:01Z","snapshot_observed_at":"2026-07-06T18:28:24.821865Z","submitted_at":"2024-06-06T18:10:11Z","title":"The Prompt Report: A Systematic Survey of Prompt Engineering Techniques","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.06608","snapshot_observed_at":"2026-08-02T03:41:06.532907Z","title":"arXiv preprint arXiv:2406.06608 , volume=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":86,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:06.532907Z"},"links":{"cited_paper":"/paper/2406.06608","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:db3be4c942613042c11ced924887d28b16354d0d2ec7e33cff914e14f8975731","observation_id":"1ac0563e-4fa9-4ef9-be9e-8daf2fca9e16","resolution":{"observed_at":"2026-08-02T03:41:06.532907Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.11931","last_updated":"2024-06-17T13:51:35Z","snapshot_observed_at":"2026-08-07T06:30:25.302107Z","submitted_at":"2024-06-17T13:51:35Z","title":"DeepSeek-Coder-V2: Breaking the Barrier of Closed-Source Models in Code Intelligence","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.11931","snapshot_observed_at":"2026-08-02T03:41:06.722239Z","title":"arXiv preprint arXiv:2406.11931 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":87,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:06.722239Z"},"links":{"cited_paper":"/paper/2406.11931","citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:2856e9f8cbe7bef2dff19ea37bb2ee1bc1851fcf8e0c00183d975fd4096161ce","observation_id":"fcaac1ac-3bfd-4fa4-bdd8-21277b31af47","resolution":{"observed_at":"2026-08-02T03:41:06.722239Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:06.967903Z","title":"2022 , eprint=","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":88,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:06.967903Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:f2ec2fe6709e4f42ed5c8c15ab07eae05c30d8bda38bf37240d67bd40ff79b52","observation_id":"6358adc9-4ed3-470e-bc4b-ae51a0f6c75f","resolution":{"observed_at":"2026-08-02T03:41:06.967903Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:07.110645Z","title":"2022 , howpublished =","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":89,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:07.110645Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:05d953b1a0a02e976c2285c8cc1b1f1d0bd2ddff8cc2768bb5dabd19d1d17116","observation_id":"1560eeaa-801d-4787-91b3-738f50d41df5","resolution":{"observed_at":"2026-08-02T03:41:07.110645Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:07.278414Z","title":"2021 , howpublished =","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":90,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:07.278414Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:0165080947e90cd9f1fa46367a4e55dd07d34f68c2e687f0245b36e358e8596d","observation_id":"5ad28b35-bab2-47d3-a96e-b9758d94c614","resolution":{"observed_at":"2026-08-02T03:41:07.278414Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:07.452672Z","title":"Empirical Software Engineering , volume=","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":91,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:07.452672Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:ee301fa93a12764a6017cb856271152b7a5122427a4bbe7d5374195bde9090b5","observation_id":"e37ec71f-ade4-46eb-8efc-d89a0023d08f","resolution":{"observed_at":"2026-08-02T03:41:07.452672Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:07.666893Z","title":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval , pages =","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":92,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:07.666893Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:751917830ce154ac4f21c5777d81499d0d2d5134375a82030c37362b40685a68","observation_id":"69407b16-ed76-4352-98df-340f2499e617","resolution":{"observed_at":"2026-08-02T03:41:07.666893Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:07.920990Z","title":"2025 , eprint=","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":93,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:07.920990Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:f8aeb9389a21bbcc03b6836c39a075ae4e3a6903cd53bd92ef6f6850ee1f3f7e","observation_id":"d60e3f7d-e07a-4d3b-837e-f5e9fa85aa1e","resolution":{"observed_at":"2026-08-02T03:41:07.920990Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:08.032344Z","title":"ToolQA: A Dataset for LLM Question Answering with External Tools , url =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":94,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:08.032344Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:8b30b0ca01bae0478165ee91a30705e04795ca9ecd52603738e543da85c73dd1","observation_id":"a0eee248-7769-4830-b340-f57da7e2c468","resolution":{"observed_at":"2026-08-02T03:41:08.032344Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:08.194423Z","title":"2024 , eprint=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":95,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:08.194423Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:f5203d6fd23c007cf2d5818da7fec45454cb499d236b79854dfb84a5b2294eca","observation_id":"8ee1e01c-d416-474a-8eed-95e9839d8040","resolution":{"observed_at":"2026-08-02T03:41:08.194423Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:08.365693Z","title":"2024 , publisher=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":96,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:08.365693Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:d3075da9c1999d4bbd3590482cf182cd0fd4418b133bc734f7befbf91d5262a1","observation_id":"f36c1a84-4bb8-40e1-b9bd-89e80020aeab","resolution":{"observed_at":"2026-08-02T03:41:08.365693Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T03:41:08.536430Z","title":"Journal of Systems and Software , volume=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models","version":1},"reference_index":97,"source":"arxiv_source","source_observed_at":"2026-08-02T03:41:08.536430Z"},"links":{"citing_paper":"/paper/2607.13820"},"observation_digest":"sha256:899e4d1c7ae0b8ef9c3b32923fcfdcbb674882d722f709523e8200bbd9ff68fb","observation_id":"51cd0afb-c0df-48d3-8770-e7dc51a85d4a","resolution":{"observed_at":"2026-08-02T03:41:08.536430Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2607.13820","last_updated":"2026-07-15T13:28:18Z","latest_version":1,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-08T03:14:24.536574Z","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models"},"reference_resolution":{"displayed":97,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":96,"verified_exact":1,"verified_fuzzy":0},"total_outbound_references":97},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"thesis":"As of 9 August 2026, this Paper Citation Record lists 97 of 97 outbound references and 1 inbound Pith citation observation for arXiv:2607.13820."}