{"as_of":"2026-08-22T17:35:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:4d5a084272974ad98d3ed0c494fb0be6c710c5b869a35b15d0e54eb07c76c4af","coverage":[{"denominator":61,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":61,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T20:41:09.087338Z","state":"measured"},{"denominator":61,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":61,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-22T06:32:14.747728+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2505.12331/citation-record","integrity":"/paper/2505.12331/integrity","json":"/paper/2505.12331/citation-record.json","paper":"/paper/2505.12331"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.762407Z","title":"Github copilot.https://copilot.github.com, 2021","venue":null,"work_id":"d9ee8c2a-d932-4158-ae9d-a086a09a7a74","year":2021},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.882885Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:188dd9ab0466e4ccd1fec4c9ea3af2d35f699ac039642d3a915d17dd10a17c14","observation_id":"421c77fd-dd77-4d41-8c94-8255de3985fa","resolution":{"observed_at":"2026-08-15T20:41:09.765896Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.752463Z","title":"Cursor: The ai-powered code editor.https://cursor.so, 2023","venue":null,"work_id":"493fcd1d-fa03-4f3a-a335-a31867d35b7e","year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.887126Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:ae05610b9c22d0f9f504a1318aaab2ae1c8cdd059d35e88d696836afc047dbb8","observation_id":"9420a2ab-6a59-4c0a-ad9c-199c6e3e0c0d","resolution":{"observed_at":"2026-08-15T20:41:09.755646Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:08.890792Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.890792Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:e652a2b32807a436f872f18bb4949292ba1a788060c1dd4f7ef7f199dee0b713","observation_id":"37a4dd52-72cb-4904-9535-2acb97562cc9","resolution":{"observed_at":"2026-08-15T20:41:08.890792Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.734430Z","title":"Codelmsec benchmark: Systematically evaluating and finding security vulnerabilities in black-box code language models","venue":null,"work_id":"8d70c3b4-cbe2-42c3-b297-3fe7bcf29fd2","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.894506Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:58f005a92621dfd592a848de4420d53eb404081134ec2fc96f9859debd5fa6bc","observation_id":"ffd97942-0d28-4e3f-8692-fa5f96fc972c","resolution":{"observed_at":"2026-08-15T20:41:09.738087Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2503.10452","last_updated":"2025-05-29T13:17:33Z","snapshot_observed_at":"2026-08-21T13:28:13.285334Z","submitted_at":"2025-03-13T15:18:56Z","title":"DynaCode: A Dynamic Complexity-Aware Code Benchmark for Evaluating Large Language Models in Code Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.10452","snapshot_observed_at":"2026-08-15T20:41:08.898224Z","title":"Dynacode: A dynamic complexity-aware code benchmark for evaluating large language models in code generation.arXiv preprint arXiv:2503.10452, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.898224Z"},"links":{"cited_paper":"/paper/2503.10452","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:69e0fb040678621d304f1410d2ed07a9aeae06a191c296be879741060175436e","observation_id":"f4362cce-2d41-45cc-9720-d8e55ab24d87","resolution":{"observed_at":"2026-08-15T20:41:08.898224Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.724381Z","title":"Humanevo: An evolution-aware benchmark for more realistic evaluation of repository-level code generation","venue":null,"work_id":"ecf143db-548b-453e-ba50-e57aa5439d62","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.902726Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:95696d29a562cec4d843ce0d9ee7832a205dae789a7540360d30099543e2f944","observation_id":"92931cb0-3c53-4536-bab2-27d63c5c177c","resolution":{"observed_at":"2026-08-15T20:41:09.728038Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:08.906493Z","title":"Codeif-bench: Evaluating instruction-following capabilities of large language models in interactive code generation.arXiv preprint arXiv:2503.22688, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.906493Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:87f5df045d255e11a39dc004e74b52567294e3c3c147e0f0cf12cf50a62bd63b","observation_id":"be0cf7b6-15bc-4966-9283-e7b95304764b","resolution":{"observed_at":"2026-08-15T20:41:08.906493Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-08-15T20:41:08.909862Z","title":"Ml-bench: Evaluating large language models and agents for machine learning tasks on repository-level code.arXiv preprint arXiv:2311.09835, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.909862Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:723ff996bfb6f40e90cd5c5fab44f0d584fd56a4bf687d4f3bd947927876adaf","observation_id":"19ebd076-1d70-427d-912f-77193fece6a3","resolution":{"observed_at":"2026-08-15T20:41:08.909862Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.06770","last_updated":"2024-11-11T23:05:04Z","snapshot_observed_at":"2026-08-18T08:11:45.716032Z","submitted_at":"2023-10-10T16:47:29Z","title":"SWE-bench: Can Language Models Resolve Real-World GitHub Issues?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.06770","snapshot_observed_at":"2026-08-15T20:41:08.913882Z","title":"Swe-bench: Can language models resolve real-world github issues?arXiv preprint arXiv:2310.06770, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.913882Z"},"links":{"cited_paper":"/paper/2310.06770","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:d3d8895add91afac0f342afb5bd2e54bfa1e42ecf4f1fe6645afae0c4a35d58f","observation_id":"0fcf29d9-48f7-44a6-bf52-a8bd0d4fd83b","resolution":{"observed_at":"2026-08-15T20:41:08.913882Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.714760Z","title":"Wang, Armando Solar-Lezama, Koushik Sen, and Ion Stoica","venue":null,"work_id":"56779f09-816c-42f6-a73d-4d18235b55c4","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.917414Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:f95d781db2631e73b80f4761f2570975f8fd75415a84b0fc571d7aae11976be6","observation_id":"be14a1d2-d442-401c-a987-89953bee9903","resolution":{"observed_at":"2026-08-15T20:41:09.718194Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.704694Z","title":null,"venue":null,"work_id":"ae9f4251-bd21-42e6-a852-043430058e04","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.920715Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:d18299ef51f2ea0c6ded94a046b97d04134241421cd47266c131c907d987c6cc","observation_id":"1f926510-652a-45e2-81e3-6297ad342f41","resolution":{"observed_at":"2026-08-15T20:41:09.708228Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2504.02605","last_updated":"2025-04-03T14:06:17Z","snapshot_observed_at":"2026-08-20T11:26:06.473228Z","submitted_at":"2025-04-03T14:06:17Z","title":"Multi-SWE-bench: A Multilingual Benchmark for Issue Resolving","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2504.02605","snapshot_observed_at":"2026-08-15T20:41:08.924377Z","title":"Multi-swe-bench: A multilingual benchmark for issue resolving, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.924377Z"},"links":{"cited_paper":"/paper/2504.02605","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:b38433ce69a1692bdb3594197895f1cfa629285ebcc4c4bb167954d5e767f81d","observation_id":"5aabb163-a1e4-4c76-8171-de64910dd20d","resolution":{"observed_at":"2026-08-15T20:41:08.924377Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.694479Z","title":"SecRepoBench: Benchmarking LLMs for secure code generation in real-world repositories, 2025","venue":null,"work_id":"fb072d00-d155-4014-ac3a-872dd26869cc","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.928133Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:1b7e72c91ccb2661f94fc6bd0399bf5504a25009d5657a8d0b51f9a09cc9a36c","observation_id":"4f899914-f0ea-4ffe-be68-ef5008893aec","resolution":{"observed_at":"2026-08-15T20:41:09.698118Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.684247Z","title":"CWEval: Outcome-driven evaluation on functionality and security of LLM code generation, 2025","venue":null,"work_id":"a23f801e-20c3-463b-9a3a-f4ea04357117","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.931450Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:25e71b21d947e0978fdce570a3d8d0dccaf61a14b50f83cf9b19bd4f6248c62d","observation_id":"175ab0dc-7966-411f-b6c0-7ddeb40f37a1","resolution":{"observed_at":"2026-08-15T20:41:09.687983Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.11470","last_updated":"2024-10-09T05:59:07Z","snapshot_observed_at":"2026-08-16T13:33:52.529038Z","submitted_at":"2024-07-16T08:08:48Z","title":"Beyond Correctness: Benchmarking Multi-dimensional Code Generation for Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.11470","snapshot_observed_at":"2026-08-15T20:41:08.934788Z","title":"Beyond correctness: Benchmarking multi-dimensional code generation for large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.934788Z"},"links":{"cited_paper":"/paper/2407.11470","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:b392067a813c9902da4cf3da55336a0be28a7e07fcfcdf549c9bbd31511dd8fa","observation_id":"d267b0db-c1a3-4ef8-b916-cecb179222a6","resolution":{"observed_at":"2026-08-15T20:41:08.934788Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.674025Z","title":"Codearena: Inspecting and improving code quality metrics using minecraft","venue":null,"work_id":"60876434-aa41-4db0-b8f5-69fcb1ef271e","year":2019},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.938516Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:920488cbd163fc29cb1a01730c06c9496373dd6c01cc58c371d41d41581b18c2","observation_id":"8097f06e-e716-4baa-a9f5-f247dd6ddd16","resolution":{"observed_at":"2026-08-15T20:41:09.677609Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.663530Z","title":"CodeElo: Benchmarking competition-level code generation of LLMs with human-comparable elo ratings, 2025","venue":null,"work_id":"dc772053-e01f-4e3c-8984-eb604bbe1fe1","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.942026Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:c5bcdcb2d0d0a2e7f92c26952c9dcfd9da53b5b2f6ee8e4f0a1aefcfd3bed88e","observation_id":"838d148b-e976-4a6a-8eae-01547b8c5a40","resolution":{"observed_at":"2026-08-15T20:41:09.667110Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.652858Z","title":"ComplexCodeEval: A benchmark for evaluating large code models on more complex code","venue":null,"work_id":"08e6eb4d-6486-4d42-8fc3-c0c27cf6073f","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.945410Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:f63368a31c98b1479efd7d6866b73ea0106c22f618822c84ed8045bab159e2e9","observation_id":"f7e97651-5964-4935-a92f-b6670019008f","resolution":{"observed_at":"2026-08-15T20:41:09.656575Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.642535Z","title":"PythonSaga: Redefining the benchmark for code generating LLMs, 2024","venue":null,"work_id":"8200056c-7fd7-4053-8d63-50c23328bd8c","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.948865Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:2feb95fa8d7f981b688d5fa2311ca4d8df6adacd9bd59292d7d4cf9e8aa5c0d8","observation_id":"d54edff5-10a6-4393-a1e0-24f00abd278f","resolution":{"observed_at":"2026-08-15T20:41:09.646182Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.18573","last_updated":"2025-03-17T17:58:13Z","snapshot_observed_at":"2026-08-16T12:59:57.442596Z","submitted_at":"2024-12-24T17:56:08Z","title":"Top General Performance = Top Domain Performance? DomainCodeBench: A Multi-domain Code Generation Benchmark","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.18573","snapshot_observed_at":"2026-08-15T20:41:08.952533Z","title":"How well do llms generate code for different application domains? benchmark and evaluation.arXiv preprint arXiv:2412.18573, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.952533Z"},"links":{"cited_paper":"/paper/2412.18573","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:a98bffc7258af1f87f2e04102ddf60eb167e228b014e39c3a6a44f4ac46ca15c","observation_id":"b56dff38-2269-4311-9d09-2d1988dba53b","resolution":{"observed_at":"2026-08-15T20:41:08.952533Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.631711Z","title":"Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, and Karthik Narasimhan","venue":null,"work_id":"2231f6ef-4dee-441f-991d-23183411ecd3","year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.956417Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:fb8bfb00f77f2d70f49f1e835fb0975bbbaee01ca3cbb75debbba5ddf65f8b2e","observation_id":"5d8a4b47-36f1-4ff6-8f67-cc13055f3cd5","resolution":{"observed_at":"2026-08-15T20:41:09.635334Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.620498Z","title":"ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation, 2023","venue":null,"work_id":"cd58e8e5-98b8-4b48-9b3b-af1d46d698d5","year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.959822Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:b0ad53ffae94e34e826361440a65cb1164d9b896268f638f6d9ce9cf43b63cde","observation_id":"f99279ee-a255-4536-9367-6778d381a976","resolution":{"observed_at":"2026-08-15T20:41:09.625092Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.609497Z","title":"HumanEval-XL: A multilingual code generation benchmark for cross-lingual natural language generalization, 2024","venue":null,"work_id":"5eccf92e-9334-4f37-aa56-78ec808f7f35","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.963333Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:8e4e84c967dcb2f090cb02fef99323d9e3d7a795d915a3e945744b593dadc5ce","observation_id":"2cbeb47d-f80b-4c4f-86fb-534d76cc5f87","resolution":{"observed_at":"2026-08-15T20:41:09.613243Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:08.966684Z","title":"Codegeex: A pre-trained model for code generation with multilingual benchmarking on humaneval-x","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.966684Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:46db9646d5d2be499c15877f6c49a5392b159e7d39afc8fde40c6e1b6b348374","observation_id":"996c0b14-7950-4375-9405-0055fdad5d9e","resolution":{"observed_at":"2026-08-15T20:41:08.966684Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:08.970147Z","title":"{AddressSanitizer}: A fast address sanity checker","venue":null,"work_id":null,"year":2012},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.970147Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:4b93a34c62c44514dae54a236e3bed89398b2b6f84cd33a36cad346c49caafe3","observation_id":"25f43123-634e-4ac1-998e-991732e8da59","resolution":{"observed_at":"2026-08-15T20:41:08.970147Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.593210Z","title":"php-src: The php interpreter.https://github.com/php/php-src, 2025","venue":null,"work_id":"7ce77d0c-dee8-4143-8265-60fd47c0a3da","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.973421Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:acdfc6b3fbeb3afdeb35ff8c576339eb3a7dfdadb32e97a02aca41065e80b958","observation_id":"74e819d9-d3f8-47ba-9843-c946e1a8f32b","resolution":{"observed_at":"2026-08-15T20:41:09.596794Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.582932Z","title":"Richard Hipp","venue":null,"work_id":"7851ace3-f84d-4a00-9a13-93adee6c61fb","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.976672Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:43a77e8f9cfa4afcc5ec314a796bbb52ef30d4a645aa82cb14553c25c9adf7ca","observation_id":"b09990d6-ebf5-434e-9d9b-0c5d93593900","resolution":{"observed_at":"2026-08-15T20:41:09.586385Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.571157Z","title":"https://testing.googleblog.com/2020/08/ code-coverage-best-practices.html, 2020","venue":null,"work_id":"75ad66af-8a97-4cc3-8618-932afe53922a","year":2020},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.979969Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:0763e1d62db1bbb667d49731760f57123b1211f512b4026fd9bb6a69d8dbfccc","observation_id":"6e377465-2058-4c79-ac59-b801173648b5","resolution":{"observed_at":"2026-08-15T20:41:09.574840Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.560897Z","title":"libclang: C interface to the clang library","venue":null,"work_id":"6a81702e-ba39-417d-99ed-34509f164e42","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.983338Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:6d7e48dc0f4ec2d3410185c596b1b991e86065da196838e0d0ea9c50fc010c1c","observation_id":"88dc70a5-32bc-440f-abeb-a04a7fac21bc","resolution":{"observed_at":"2026-08-15T20:41:09.564370Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:08.987369Z","title":null,"venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.987369Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:d572b9208a601ad75d46ac4cdd79ceb8222e46f812b50866c16c1a0b5966f72e","observation_id":"d1450324-a4f6-45df-a943-c1b8d6b4b472","resolution":{"observed_at":"2026-08-15T20:41:08.987369Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.544190Z","title":"difflib — helpers for computing deltas between objects","venue":null,"work_id":"71acb22c-2ee3-4ab9-b0de-d6aab1f6118b","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.990624Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:d34e7ba746e96e20ebc597bf690ceca65ed7e90f7431ede2dd17526aeaa504fa","observation_id":"1978e6ba-1cb1-4ecc-bc52-25c65abf6ff3","resolution":{"observed_at":"2026-08-15T20:41:09.548044Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.534683Z","title":"gpt-o1 model.https://platform.openai.com/docs/models/o1, 2025","venue":null,"work_id":"4ed32e90-4dfa-491e-aa1a-7e1954960f95","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.993964Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:66ff2cf719298271285e545269a163caef5284e675bd8e1695d5446a5c9e3b7e","observation_id":"cd2d49f0-5dc8-4fae-98e3-42c38be67156","resolution":{"observed_at":"2026-08-15T20:41:09.538208Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.525080Z","title":"o3-mini model.https://platform.openai.com/docs/models/o3-mini, 2025","venue":null,"work_id":"02796f5c-9207-4c38-807f-fd8b97af4839","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.997232Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:70bf5e7d68fb1c14c15e22ac20b043aabf6c1d9f299fec4b30d5a47f722c5e85","observation_id":"ab3f1842-84bd-444f-b18d-d960ecb00322","resolution":{"observed_at":"2026-08-15T20:41:09.528623Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.515439Z","title":"Claude 3.7 sonnet.https://www.anthropic.com/claude/sonnet, 2025","venue":null,"work_id":"aa9e2f17-1f7c-403c-83c8-af02e399ab0f","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.000387Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:6d55d2a0a73fca10d0b85ab403f1d205b19bc84075ab93c4491e1280d67e3687","observation_id":"c5d584d7-b454-482d-bd96-1006e253fffb","resolution":{"observed_at":"2026-08-15T20:41:09.518845Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.505879Z","title":"Claude 3.5 haiku.https://www.anthropic.com/claude/haiku, 2025","venue":null,"work_id":"7561c8a6-2043-4fe9-9549-825687e7ef92","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.003876Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:37ac304e22fce2786b9ccab9814e0b19f2ff6a73a844beafe9786ab0d3260577","observation_id":"5ac39b2b-67e2-42b1-a827-d6712bfb99da","resolution":{"observed_at":"2026-08-15T20:41:09.509216Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.495872Z","title":"Gemini 2.5 flash.https://developers.generativelanguage.google/, 2025","venue":null,"work_id":"e0262a86-601a-45e4-9947-8fe5fae39e69","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.007301Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:4efa641e18c22da95bf6346da8675db9c100bea3c50ed7126feffd7e34f3c6c8","observation_id":"ecb39fce-e3dc-4376-b46c-2dc6b95967f7","resolution":{"observed_at":"2026-08-15T20:41:09.499364Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.486152Z","title":"Llama 3.3 70b instruct (fp16)","venue":null,"work_id":"a2ae3316-da58-4e0e-8e00-f98543aef075","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.010502Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:8c7a9c62dabba4005f1189201d904ada7695619ee9bc7d85e2401b71f72e2c46","observation_id":"614a4c67-f17b-462e-afc7-18d9722d4bb9","resolution":{"observed_at":"2026-08-15T20:41:09.489480Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.476608Z","title":"Codellama 70b instruct (fp16)","venue":null,"work_id":"13b9c6cb-465f-4bb3-9e5a-a4ff8fe236b6","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.013703Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:dd36efc0ed9040568553ee7129e06ccee0e928401dd00af960ca199c0e74857c","observation_id":"3e19ca2f-c7eb-4839-9edb-39d90a5268ed","resolution":{"observed_at":"2026-08-15T20:41:09.479853Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.466903Z","title":"Qwen 2.5 coder 32b instruct (fp16)","venue":null,"work_id":"40e678b2-3366-4413-91fb-f0259d6ff7b5","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.016914Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:507cd3954f2e29f31768c179f99fd57a71ef8ad5f032d1519b7b063422160bdd","observation_id":"d97379d9-4aec-40a6-af08-2b98a80f12bc","resolution":{"observed_at":"2026-08-15T20:41:09.470476Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.456834Z","title":"Qwen 3.0 30b-a3b fp16.https://ollama.com/library/qwen3:30b-a3b-fp16, 2025","venue":null,"work_id":"11470e59-eb75-4ab9-9862-6ec3581c4be6","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.020235Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:4a79b96f2d9d3f478c7629589c85791088c183ce1fd5270581ddf278c138e2ae","observation_id":"16c0acef-3449-42ba-8439-7306efa9a235","resolution":{"observed_at":"2026-08-15T20:41:09.460304Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.447220Z","title":"Qwen 3 8b fp16.https://ollama.com/library/qwen3:8b-fp16, 2025","venue":null,"work_id":"9dc3ec90-0621-47ed-a55b-d86741a8177f","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.023486Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:428497e4f86d30d291a1b8bbd65b6f813c2b7328d68b487ce5e89462584cff01","observation_id":"36fdb130-8cbe-47e1-ad88-2d20377819a9","resolution":{"observed_at":"2026-08-15T20:41:09.450714Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.437295Z","title":"Gemma 3 27b-it fp16.https://ollama.com/library/gemma3:27b-it-fp16, 2025","venue":null,"work_id":"731b3efe-4114-473b-b160-334c019eecf4","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.026695Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:0a056a68e4dd9ee42caf92405ded6560337b4386b4e6b108448e9505d0378801","observation_id":"ed415a7d-841a-4574-9f59-df739406e666","resolution":{"observed_at":"2026-08-15T20:41:09.440770Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.427673Z","title":"Qwen 2.5 coder 14b instruct (fp16)","venue":null,"work_id":"dc764a59-fc37-46ec-b2b1-997c675ce47d","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.029928Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:9f6f1a4bf07c1597081d9d34ce86214e102c09a21b4a7e43e2560e41cdba3aa3","observation_id":"f14f87fd-c5ed-4fa4-9171-ec76af6b46ed","resolution":{"observed_at":"2026-08-15T20:41:09.430956Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.418038Z","title":"Deepseek coder v2 16b lite instruct (fp16)","venue":null,"work_id":"d4368407-6fc7-45ec-91f1-6e01cb845677","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.033335Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:f66b88a642dc29f0a625a7d9ef5ac5afbd6212d8570646b593c65cd2e672e231","observation_id":"2e09b457-c428-4822-8ef3-01b67b7c49c2","resolution":{"observed_at":"2026-08-15T20:41:09.421381Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.408557Z","title":"Starcoder2-15b-instruct-v0.1 (fp16)","venue":null,"work_id":"d08ebbe6-0a08-41e1-96a2-4a9e447ea5de","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.036683Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:e8726357f33770092ffadbe1f50f567e1bb63bcd2a65b217c47048ee38d26c5d","observation_id":"2651a8ef-fb93-47ae-a2dc-546876a5fc10","resolution":{"observed_at":"2026-08-15T20:41:09.411806Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.399435Z","title":"Phi-4 14b fp16.https://ollama.com/library/phi4:14b-fp16, 2025","venue":null,"work_id":"8d8a273c-4f94-470f-8a40-0d8cf119e0fd","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.039967Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:a69ee783965558ee42d596fad5f4e56d8b81eb901b4618506c7962afc39d4463","observation_id":"065a418e-8f61-4bad-9e28-8c86c4b37fce","resolution":{"observed_at":"2026-08-15T20:41:09.402729Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.389546Z","title":"Mistral 7b instruct (fp16)","venue":null,"work_id":"374430d6-0a45-4822-bef5-f8131a54bbfd","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.043072Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:456d3e104ae8dd155d7765e53f8b2666821f40b70c95a8400c4cd0b77dd87e14","observation_id":"e8a8fc19-035d-41bc-9fb8-27aa8ccfdf02","resolution":{"observed_at":"2026-08-15T20:41:09.392942Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.379087Z","title":"Codegemma 7b instruct (fp16)","venue":null,"work_id":"d3f285fb-af88-45e0-a7c8-fa4c60bbf277","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.045955Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:c33177fb1a81c0d2e8bd2dce4758c16af9cd98f84ac753d9afcc1d675196faf6","observation_id":"56c62f9f-203a-46fc-bb32-8424f361b066","resolution":{"observed_at":"2026-08-15T20:41:09.382442Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.369252Z","title":"Openai: Advances in safe and beneficial ai.https://openai.com, 2025","venue":null,"work_id":"f5598d5f-504f-44c6-8e4a-fcec737a043c","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.048955Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:1ed5c3d701664fac47d9f15caeb78dc6fe7dc746109c4d4ff409878990e39cb4","observation_id":"51c00e5f-a141-4348-b0c7-1541b8d6090b","resolution":{"observed_at":"2026-08-15T20:41:09.372657Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.358847Z","title":"Anthropic: Building reliable, steerable ai systems","venue":null,"work_id":"92d76da9-2e32-41c8-a171-373337a9d36b","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.052241Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:994f76d5ba0ebd1a12c800707a7ea3f0c0744213e0c898b2b210aba70e1e7a66","observation_id":"1074e2ff-727f-496b-800a-30dcd2e6b1b3","resolution":{"observed_at":"2026-08-15T20:41:09.362447Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.348857Z","title":"Google: Organizing the world’s information.https://www.google.com, 2025","venue":null,"work_id":"13dfaf51-b149-473f-9edb-615b58078fd9","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.055397Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:3ad050946c00a5734d9495929c7a4ff9520af5d78a806e2d242995bed0c7d19b","observation_id":"faa5be50-d86b-462e-97bc-7f4e4e873a70","resolution":{"observed_at":"2026-08-15T20:41:09.352265Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.338927Z","title":"Deepseek: Developer of high-performance open-source llms","venue":null,"work_id":"e9c32ac2-acc9-4600-9a90-cc12c38650d2","year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.058832Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:6805b34cc3011f321f7de711356aa9654dbc516ab8d053b2faae26e4036ad528","observation_id":"b029a42d-0535-4919-b5d9-939351530fc5","resolution":{"observed_at":"2026-08-15T20:41:09.342614Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.328286Z","title":"Alibaba group: Global trade and technology.https://www.alibaba.com, 2025","venue":null,"work_id":"40a8c1cb-5b1f-49b1-ada6-2af688814875","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.061970Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:3b1d096b8d75e0d9b6305c04cc29de64793ba4c9fff0b90adf007934983abb06","observation_id":"c12238fa-e33f-4420-ac49-bcf5605bd33e","resolution":{"observed_at":"2026-08-15T20:41:09.332122Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.317399Z","title":"Meta: Bringing the metaverse and social technology together","venue":null,"work_id":"c53bf76b-4a4c-4741-ba9b-e743306c13db","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.065031Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:77928c511a7ca56f2e012a82ee7fd42d467ceb029b6dcab83b2181b17a63aea9","observation_id":"c9f9aa06-2519-4215-acfd-e96bd8cb4b54","resolution":{"observed_at":"2026-08-15T20:41:09.321033Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.306746Z","title":"Microsoft: Empowering every person and organization","venue":null,"work_id":"9d55bcc0-fe0d-45e9-a6ac-652173b0b644","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.068195Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:ef8b9a6ff77981e2520e79d22ecb1a0898cb93e65eefc93b797ce39423feb1c4","observation_id":"cded8149-9ca2-4f3e-80e8-5a24d16e8f85","resolution":{"observed_at":"2026-08-15T20:41:09.310513Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.296643Z","title":"Bigcode: Open and responsible development of code llms","venue":null,"work_id":"879e4e8b-f773-4500-8b29-b669bcbce97c","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.071519Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:a9383d14d2efa3734f52c6bcd95864cebd4e5e0bf1ea9f9922e90ec20a276332","observation_id":"1ab6e7d1-79b9-4dc6-904f-a9092340bc36","resolution":{"observed_at":"2026-08-15T20:41:09.299952Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.286313Z","title":"Mistral ai: Frontier ai in your hands.https://mistral.ai, 2025","venue":null,"work_id":"01b91de0-eb65-49d5-9323-6b2b58eab765","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.074554Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:4444dc665c67718e8e93676307c9a2bbcb83edf8e218a71dc4c70b3ac68186d4","observation_id":"2c5f0684-a7fa-4f47-ab91-988aacd0d958","resolution":{"observed_at":"2026-08-15T20:41:09.289957Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:41:09.275815Z","title":"Ollama: Get up and running with large language models locally","venue":null,"work_id":"3ac8d607-252b-4038-a5cc-9be9eed47ecf","year":2025},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.077778Z"},"links":{"citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:f866afe6ee855909953a229cb8866d7839fbc8fd84667ae15c17045523d87f3f","observation_id":"77e66e68-6bcb-4ffd-a89b-8398e909a334","resolution":{"observed_at":"2026-08-15T20:41:09.279340Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1906.04908","last_updated":"2019-06-12T03:13:18Z","snapshot_observed_at":"2026-08-19T00:17:44.944869Z","submitted_at":"2019-06-12T03:13:18Z","title":"SPoC: Search-based Pseudocode to Code","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1906.04908","snapshot_observed_at":"2026-08-15T20:41:09.080649Z","title":"Spoc: Search-based pseudocode to code","venue":null,"work_id":null,"year":1906},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.080649Z"},"links":{"cited_paper":"/paper/1906.04908","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:62c7814588bed830141481b7f5142f38252dadee7f7e57113ea62d7be323c9b4","observation_id":"6376ec4b-63b3-4a79-bb6d-4e07616eb100","resolution":{"observed_at":"2026-08-15T20:41:09.080649Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-20T20:50:23.483838Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-15T20:41:09.084092Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.084092Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:97015d11dc395c6e326ec9b6d64c0c7b17a15bd5b9371c8a2623aaacdf923d4e","observation_id":"c79ca41e-c891-4d6a-b74f-17d8593de675","resolution":{"observed_at":"2026-08-15T20:41:09.084092Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.21713","last_updated":"2025-02-04T03:02:58Z","snapshot_observed_at":"2026-08-16T13:05:07.586558Z","submitted_at":"2024-10-29T03:54:59Z","title":"Fuzzing the PHP Interpreter via Dataflow Fusion","version":2},"cited_work":{"arxiv_id":"2410.21713","doi":null,"metadata_source":"pith","pith_arxiv_id":"2410.21713","snapshot_observed_at":"2026-08-15T20:41:09.123046Z","title":"Fuzzing the PHP Interpreter via Dataflow Fusion","venue":"cs.CR","work_id":"e98cfdd8-89d6-4000-8951-ad7fce986284","year":2024},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:09.087338Z"},"links":{"cited_paper":"/paper/2410.21713","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:0299a4ed7379c9dd80e5ccd8ada4fe8d9a6eb53e1370f1866ec7fc44ebd26160","observation_id":"ca45235d-09f5-492e-9102-3bbbf33dedfd","resolution":{"observed_at":"2026-08-15T20:41:09.129077Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","latest_version":2,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs"},"reference_resolution":{"displayed":61,"state_counts":{"malformed_identifier":0,"metadata_mismatch":1,"parse_uncertain":0,"unresolved":14,"verified_exact":0,"verified_fuzzy":46},"total_outbound_references":61},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"thesis":"As of 22 August 2026, this Paper Citation Record lists 61 of 61 outbound references and 0 inbound Pith citation observations for arXiv:2505.12331."}