{"as_of":"2026-08-23T16:15:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:5b0268b7f4035f8047eb513c17f365d952d2186d3294ba0e0bc7554f3c6853d5","coverage":[{"denominator":52,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":52,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T21:47:07.814079Z","state":"measured"},{"denominator":55,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":55,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-23T06:30:58.430688+00:00","state":"measured"},{"denominator":3,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":3,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T14:15:49.870701Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-02T21:47:27.669855Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2505.09027","snapshot_observed_at":"2026-08-07T14:15:49.870701Z","title":"Tests as prompt: A test-driven-development benchmark for llm code generation,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.19443","last_updated":"2025-05-26T03:00:21Z","snapshot_observed_at":"2026-08-20T00:20:09.989630Z","submitted_at":"2025-05-26T03:00:21Z","title":"Vibe Coding vs. Agentic Coding: Fundamentals and Practical Implications of Agentic AI","version":1},"reference_index":191,"source":"pdf_text","source_observed_at":"2026-08-07T14:15:49.870701Z"},"links":{"cited_paper":"/paper/2505.09027","citing_paper":"/paper/2505.19443"},"observation_digest":"sha256:1b0762f2056b489c9187c44b6fdfca43bcba432098d88cdc4897a23cfab1d95f","observation_id":"f9739363-1866-4844-9794-a79322847282","resolution":{"observed_at":"2026-08-07T14:15:49.870701Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"cited_work":{"arxiv_id":"2505.09027","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2505.09027","snapshot_observed_at":"2026-07-02T21:47:27.669855Z","title":null,"venue":null,"work_id":"67bc9a68-e9a8-4103-b337-1f4460a121cc","year":2025},"citing_paper":{"arxiv_id":"2604.26615","last_updated":"2026-04-29T12:43:22Z","snapshot_observed_at":"2026-08-15T06:59:13.860694Z","submitted_at":"2026-04-29T12:43:22Z","title":"TDD Governance for Multi-Agent Code Generation via Prompt Engineering","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-05-07T11:40:59.915105Z"},"links":{"cited_paper":"/paper/2505.09027","citing_paper":"/paper/2604.26615"},"observation_digest":"sha256:da32930f4b31fe1268f7678e177b6ea862f1623bd9bdc9ac4c736bd49f78329c","observation_id":"9a7f6292-c31c-4d4b-a303-b8ef39fabb6b","resolution":{"observed_at":"2026-05-12T09:16:26.986082Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"cited_work":{"arxiv_id":"2505.09027","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2505.09027","snapshot_observed_at":"2026-07-02T21:47:27.669855Z","title":null,"venue":null,"work_id":"67bc9a68-e9a8-4103-b337-1f4460a121cc","year":2025},"citing_paper":{"arxiv_id":"2606.08135","last_updated":"2026-06-06T12:23:30Z","snapshot_observed_at":"2026-08-16T20:41:24.242908Z","submitted_at":"2026-06-06T12:23:30Z","title":"TICoder: A Repository-Level Code Generation Framework with Test-Driven Planning and Implementation-Aware Reuse","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-06-27T19:33:55.542004Z"},"links":{"cited_paper":"/paper/2505.09027","citing_paper":"/paper/2606.08135"},"observation_digest":"sha256:c96c5b907a8d760901e2933b0299a25ecdf465edaea5571debee8b78eefc68e4","observation_id":"599f82cd-22cb-4a15-ae19-d5bf46f4818e","resolution":{"observed_at":"2026-07-02T21:47:27.671119Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2505.09027/citation-record","integrity":"/paper/2505.09027/integrity","json":"/paper/2505.09027/citation-record.json","paper":"/paper/2505.09027"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:08.679908Z","title":"https://fireship.io/, 2017","venue":null,"work_id":"012e8f5f-d971-4208-a7e0-a5ef23c7c6d0","year":2017},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.575875Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:7a3dbc92b85f9623742d8a16471f0ed4b9d8a5632b4f3e0e6f8adff27b454332","observation_id":"0462b0fe-f9b9-449a-876d-c752e7a6d3b4","resolution":{"observed_at":"2026-08-15T21:47:08.684178Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:08.666860Z","title":"https://huggingface.co/spaces/onekq-ai/WebApp1K-models-leaderboard, 2024","venue":null,"work_id":"43e4ac24-1374-4c2e-aa60-25924294898a","year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.581244Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:474dac6d6b18c3cfd3bcef1704d402d1f607857e9e46a9ddf2bf2c7ace406a19","observation_id":"3650eac4-9627-493e-9f51-7021453ed6d3","resolution":{"observed_at":"2026-08-15T21:47:08.670920Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:08.653934Z","title":"Fullstack React: The Complete Guide to ReactJS and Friends","venue":null,"work_id":"7348731e-fd13-44f5-8c9b-6d03623ead0e","year":2017},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.586171Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:7fc3df8996702ec2c4f4f853e27fbe3d0ab8149e6b5ff56e2cea82aaabcefb09","observation_id":"dd21de74-1115-4612-92d6-aae255055870","resolution":{"observed_at":"2026-08-15T21:47:08.658109Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:08.639769Z","title":"W., Tian, Z., and Barber, D","venue":null,"work_id":"4fa4c9c6-a24f-43e6-8b53-c5ed4a6a0c91","year":2017},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.590592Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:129467b3d78eccecfd201a9edd8d2a6db3fb23f919acda45eeea02b422ba7468","observation_id":"0d262345-6a76-4e2c-a032-ac6fb2a700c5","resolution":{"observed_at":"2026-08-15T21:47:08.644106Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2108.07732","last_updated":"2021-08-16T03:57:30Z","snapshot_observed_at":"2026-08-15T17:40:38.050939Z","submitted_at":"2021-08-16T03:57:30Z","title":"Program Synthesis with Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.07732","snapshot_observed_at":"2026-08-15T21:47:07.594998Z","title":"Program synthesis with large language models","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.594998Z"},"links":{"cited_paper":"/paper/2108.07732","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:63e5cd10ba2081f6b500b8997d91d33302455081861e7b0eb572dd7ad606987f","observation_id":"3ff65f97-89f7-4589-ac24-76abb06bcb5f","resolution":{"observed_at":"2026-08-15T21:47:07.594998Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:08.626261Z","title":"Test Driven Development: By Example","venue":null,"work_id":"7fcf3d6b-81e7-401b-bd38-2dde100efe3f","year":2022},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.599971Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:48b22517e9c279619a6adfa47796e7991acb96893c33a504178e0df8d6bee3af","observation_id":"720d92f7-4a2a-4aaa-9586-9164d968b40d","resolution":{"observed_at":"2026-08-15T21:47:08.630485Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-20T20:50:23.483838Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-15T21:47:07.605847Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.605847Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:9ef802930075665e3121d5ec45935d94f45d23dac11df0df5e258dbe4819225f","observation_id":"76ea2de5-545f-4994-994b-648d53a26076","resolution":{"observed_at":"2026-08-15T21:47:07.605847Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2301.08721","last_updated":"2023-10-24T07:58:35Z","snapshot_observed_at":"2026-08-18T00:00:51.362312Z","submitted_at":"2023-01-19T02:29:23Z","title":"Batch Prompting: Efficient Inference with Large Language Model APIs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2301.08721","snapshot_observed_at":"2026-08-15T21:47:07.610338Z","title":"Batch prompting: Efficient inference with large language model apis, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.610338Z"},"links":{"cited_paper":"/paper/2301.08721","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:416e9ca54a41145fa991ed4a90e0515251e8dc51c435795708675b2da19b125c","observation_id":"a0584331-f6f9-4d53-872d-dae2d6bb9602","resolution":{"observed_at":"2026-08-15T21:47:07.610338Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.04757","last_updated":"2023-06-15T05:08:56Z","snapshot_observed_at":"2026-08-16T15:25:41.826330Z","submitted_at":"2023-06-07T20:12:29Z","title":"INSTRUCTEVAL: Towards Holistic Evaluation of Instruction-Tuned Large Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.04757","snapshot_observed_at":"2026-08-15T21:47:07.615092Z","title":"K., Hong, P., Bing, L., and Poria, S","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.615092Z"},"links":{"cited_paper":"/paper/2306.04757","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:f59ca8de163b168f9e2484174eac67c077f5b08a33ef7139b1242295d2632bbd","observation_id":"04b4a062-6245-4b85-926f-29b4690499bb","resolution":{"observed_at":"2026-08-15T21:47:07.615092Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.12948","last_updated":"2026-01-04T03:57:36Z","snapshot_observed_at":"2026-08-15T12:33:55.451951Z","submitted_at":"2025-01-22T15:19:35Z","title":"DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.12948","snapshot_observed_at":"2026-08-15T21:47:07.620033Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.620033Z"},"links":{"cited_paper":"/paper/2501.12948","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:35bc69b38aff937ae1e9e1b452286c88e7f12c840d9e4e4898dd2480905ee8a8","observation_id":"719c4ffa-361f-4604-a5ce-d629d2676e6d","resolution":{"observed_at":"2026-08-15T21:47:07.620033Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2301.00234","last_updated":"2024-10-05T11:47:02Z","snapshot_observed_at":"2026-07-06T14:36:25.690733Z","submitted_at":"2022-12-31T15:57:09Z","title":"A Survey on In-context Learning","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2301.00234","snapshot_observed_at":"2026-08-15T21:47:07.624728Z","title":"A survey on in-context learning, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.624728Z"},"links":{"cited_paper":"/paper/2301.00234","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:b3de569a11235128ad07136b1aba34f38137ca95684e5169bb7f5883f9565279","observation_id":"6a0f1859-90db-4a62-83c0-739ebb42685c","resolution":{"observed_at":"2026-08-15T21:47:07.624728Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.01861","last_updated":"2023-08-14T09:07:00Z","snapshot_observed_at":"2026-08-19T20:54:53.128399Z","submitted_at":"2023-08-03T16:31:02Z","title":"ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.01861","snapshot_observed_at":"2026-08-15T21:47:07.629346Z","title":"Classeval: A manually-crafted benchmark for evaluating llms on class-level code generation","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.629346Z"},"links":{"cited_paper":"/paper/2308.01861","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:9b97614018c0521f6b52ac0d307aaef33380ccac0005688107aab27d38faf1b1","observation_id":"461be28f-c97b-4641-9370-7ef4f096f8d3","resolution":{"observed_at":"2026-08-15T21:47:07.629346Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.15363","last_updated":"2023-11-20T13:59:16Z","snapshot_observed_at":"2026-08-19T21:06:13.923905Z","submitted_at":"2023-08-29T14:59:54Z","title":"Text-to-SQL Empowered by Large Language Models: A Benchmark Evaluation","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.15363","snapshot_observed_at":"2026-08-15T21:47:07.634038Z","title":"Text-to-sql empowered by large language models: A benchmark evaluation, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.634038Z"},"links":{"cited_paper":"/paper/2308.15363","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:1c1814659ff2dd1c17ace778c16468797afb5703b6095175495ae381fca4a6a1","observation_id":"fd1fbb5f-f044-4772-8892-8889ee8514fc","resolution":{"observed_at":"2026-08-15T21:47:07.634038Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.14254","last_updated":"2024-09-21T22:36:22Z","snapshot_observed_at":"2026-08-18T13:44:27.317595Z","submitted_at":"2024-09-21T22:36:22Z","title":"Instruction Following without Instruction Tuning","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.14254","snapshot_observed_at":"2026-08-15T21:47:07.638694Z","title":"F., Liang, P., and Manning, C","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.638694Z"},"links":{"cited_paper":"/paper/2409.14254","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:d1c5ea4f92f85f618b0d5f0d13250c6ea63368d0e8608cbc3bf036f9e924f3af","observation_id":"9c5cb4a5-f725-440f-a4cf-25c7d7ffcea3","resolution":{"observed_at":"2026-08-15T21:47:07.638694Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.04905","last_updated":"2025-03-20T03:28:56Z","snapshot_observed_at":"2026-08-21T08:25:18.035774Z","submitted_at":"2024-11-07T17:47:25Z","title":"OpenCoder: The Open Cookbook for Top-Tier Code Large Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.04905","snapshot_observed_at":"2026-08-15T21:47:07.643347Z","title":"K., Hao, J., Song, L., Xu, Y., Yang, J., Liu, J","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.643347Z"},"links":{"cited_paper":"/paper/2411.04905","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:d7242c4b31106587b65a28857fb76b612cda78ba8ded76dbbbecc1facd288d41","observation_id":"74e871a1-6597-4553-9e42-b263a33f2bae","resolution":{"observed_at":"2026-08-15T21:47:07.643347Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.12186","last_updated":"2024-11-12T13:24:25Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-09-18T17:57:57Z","title":"Qwen2.5-Coder Technical Report","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.12186","snapshot_observed_at":"2026-08-15T21:47:07.648125Z","title":"Qwen2.5-coder technical report, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.648125Z"},"links":{"cited_paper":"/paper/2409.12186","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:90bbee60dee1983a3d5e55e2e2e2d62e1e87d649ab9ebe4eab403371edd05b9a","observation_id":"87ec06e8-8f97-4d0a-ac33-31ea501dc86e","resolution":{"observed_at":"2026-08-15T21:47:07.648125Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.07031","last_updated":"2024-01-30T20:50:56Z","snapshot_observed_at":"2026-08-16T14:27:46.619038Z","submitted_at":"2024-01-13T10:19:26Z","title":"Code Security Vulnerability Repair Using Reinforcement Learning with Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.07031","snapshot_observed_at":"2026-08-15T21:47:07.652728Z","title":"T., Karkevandi, M","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.652728Z"},"links":{"cited_paper":"/paper/2401.07031","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:cea2f6f5d34322812c77cfa1e7b995af0713c14a07899ce419790f16b6690eb0","observation_id":"e64c5eb5-2b86-49d0-8479-78cff52564e6","resolution":{"observed_at":"2026-08-15T21:47:07.652728Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.03374","last_updated":"2024-02-22T00:29:37Z","snapshot_observed_at":"2026-08-16T14:29:22.541271Z","submitted_at":"2024-01-07T02:46:39Z","title":"LLM-Powered Code Vulnerability Repair with Reinforcement Learning and Semantic Reward","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.03374","snapshot_observed_at":"2026-08-15T21:47:07.657348Z","title":"T., Khoury, J., Seong, A., Karkevandi, M","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.657348Z"},"links":{"cited_paper":"/paper/2401.03374","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:788dfd2d3da31179ceeea9a815888905cba14f0125727d3fa2637cef623620b5","observation_id":"d9525066-813c-4d68-9fb3-4c8eb248a001","resolution":{"observed_at":"2026-08-15T21:47:07.657348Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.18341","last_updated":"2023-12-23T20:00:52Z","snapshot_observed_at":"2026-08-16T15:29:32.225038Z","submitted_at":"2023-05-25T22:09:08Z","title":"Coarse-Tuning Models of Code with Reinforcement Learning Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.18341","snapshot_observed_at":"2026-08-15T21:47:07.662160Z","title":"Coarse-tuning models of code with reinforcement learning feedback, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.662160Z"},"links":{"cited_paper":"/paper/2305.18341","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:89d382df48ceb6116ff10868b6f96d6018e07cdc0140a68c671ff61360399c05","observation_id":"defd3581-e110-4905-9dc0-8d4a0f74517c","resolution":{"observed_at":"2026-08-15T21:47:07.662160Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.20410","last_updated":"2024-06-05T15:39:26Z","snapshot_observed_at":"2026-08-21T03:50:34.502633Z","submitted_at":"2023-10-31T12:32:38Z","title":"FollowBench: A Multi-level Fine-grained Constraints Following Benchmark for Large Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.20410","snapshot_observed_at":"2026-08-15T21:47:07.666667Z","title":"Followbench: A multi-level fine-grained constraints following benchmark for large language models, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.666667Z"},"links":{"cited_paper":"/paper/2310.20410","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:d4a3e46919cc7e8795a78e71f6b586bf4a03175aa5a19861ed1e92b907611466","observation_id":"c69ee599-1f0a-498a-a851-125011ea847d","resolution":{"observed_at":"2026-08-15T21:47:07.666667Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:08.612862Z","title":"E., Yang, J., Wettig, A., Yao, S., Pei, K., Press, O., and Narasimhan, K","venue":null,"work_id":"e14c69da-8456-497d-952e-162e5081ea31","year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.671054Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:714cdf80bd30bbf2cc34a88804876d18b44c69f082fcddc3bf695839420ad3b8","observation_id":"03882f12-0034-442f-8a59-4e8313df98a2","resolution":{"observed_at":"2026-08-15T21:47:08.617178Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.11501","last_updated":"2022-11-18T17:20:27Z","snapshot_observed_at":"2026-08-16T16:15:02.405696Z","submitted_at":"2022-11-18T17:20:27Z","title":"DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.11501","snapshot_observed_at":"2026-08-15T21:47:07.675498Z","title":"W., Fried, D., Wang, S., and Yu, T","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.675498Z"},"links":{"cited_paper":"/paper/2211.11501","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:650255397d5a5b0c4c68389efd3799c8d50a7e5cf16f8648545ad6babbc70fc8","observation_id":"4adbef44-8446-42d9-8d7a-f93b43b868dd","resolution":{"observed_at":"2026-08-15T21:47:07.675498Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.06161","last_updated":"2023-12-13T14:44:10Z","snapshot_observed_at":"2026-07-06T15:25:35.930688Z","submitted_at":"2023-05-09T08:16:42Z","title":"StarCoder: may the source be with you!","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.06161","snapshot_observed_at":"2026-08-15T21:47:07.680056Z","title":"B., Zi, Y., Muennighoff, N., Kocetkov, D., Mou, C., Marone, M., Akiki, C., Li, J., Chim, J., Liu, Q., Zheltonozhskii, E., Zhuo, T","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.680056Z"},"links":{"cited_paper":"/paper/2305.06161","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:f35ed69a4cca48a771ccb22ce95434c3116c8120af203da32492f74e655b2850","observation_id":"915a6cd4-65f3-4ef4-a628-341ea80bc722","resolution":{"observed_at":"2026-08-15T21:47:07.680056Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.14660","last_updated":"2025-02-25T14:49:33Z","snapshot_observed_at":"2026-08-21T19:30:13.639538Z","submitted_at":"2024-05-23T14:57:52Z","title":"Implicit In-context Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.14660","snapshot_observed_at":"2026-08-15T21:47:07.684338Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.684338Z"},"links":{"cited_paper":"/paper/2405.14660","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:353a16d5d96f96fb29d0ae2db8d88830fe28b06f4769b25f1e5b45193c18d985","observation_id":"9fa98f54-f811-429b-a87d-1c8a57f09ffe","resolution":{"observed_at":"2026-08-15T21:47:07.684338Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.20050","last_updated":"2023-05-31T17:24:00Z","snapshot_observed_at":"2026-08-17T09:42:34.746112Z","submitted_at":"2023-05-31T17:24:00Z","title":"Let's Verify Step by Step","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.20050","snapshot_observed_at":"2026-08-15T21:47:07.688878Z","title":"Let's verify step by step, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.688878Z"},"links":{"cited_paper":"/paper/2305.20050","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:79755bcfde8c95e1372d2026d2d13793f1ed61c7e5145f435729c634e7b37475","observation_id":"ed84cfbb-69f1-41ea-ab58-a338c2f77e91","resolution":{"observed_at":"2026-08-15T21:47:07.688878Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:08.599506Z","title":"F., Lin, K., Hewitt, J., Paranjape, A., Bevilacqua, M., Petroni, F., and Liang, P","venue":null,"work_id":"dcfb0613-851d-469a-a9fe-bff6b2445ca3","year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.693330Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:b8edb06cdd6f61bf41f9a21b41add40714229e2f93be19f39f36f86ec0241442","observation_id":"a8ea5523-8453-4789-bad1-2f6484f8211b","resolution":{"observed_at":"2026-08-15T21:47:08.603780Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2303.10475","last_updated":"2024-05-25T03:21:21Z","snapshot_observed_at":"2026-08-19T10:53:16.609842Z","submitted_at":"2023-03-18T19:17:47Z","title":"Large Language Model Instruction Following: A Survey of Progresses and Challenges","version":8},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.10475","snapshot_observed_at":"2026-08-15T21:47:07.697713Z","title":"Large language model instruction following: A survey of progresses and challenges, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.697713Z"},"links":{"cited_paper":"/paper/2303.10475","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:81db1c149d78f60013dc98b9e31a0bf8467bdeda723667146221078886eb82a9","observation_id":"c97fe7e4-7e51-47c6-9df7-1b3c0cff747c","resolution":{"observed_at":"2026-08-15T21:47:07.697713Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.19173","last_updated":"2024-02-29T13:53:35Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-02-29T13:53:35Z","title":"StarCoder 2 and The Stack v2: The Next Generation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.19173","snapshot_observed_at":"2026-08-15T21:47:07.702234Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.702234Z"},"links":{"cited_paper":"/paper/2402.19173","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:27f8785855531947b26785ed071f9aee0bb888cb993e08d29b1e3ba57a820364","observation_id":"d2b2882a-a8e7-4d3a-8d5b-abc2d9b55254","resolution":{"observed_at":"2026-08-15T21:47:07.702234Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.13521","last_updated":"2024-06-11T15:53:35Z","snapshot_observed_at":"2026-08-16T14:16:44.420948Z","submitted_at":"2024-02-21T04:10:12Z","title":"Test-Driven Development for Code Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.13521","snapshot_observed_at":"2026-08-15T21:47:07.706561Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.706561Z"},"links":{"cited_paper":"/paper/2402.13521","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:c91dd0a09945fff47a77e1c3d19ea03d6105dbcd53c6534127612c0571107ceb","observation_id":"df2d2159-68ab-4370-a72f-070494b27d7b","resolution":{"observed_at":"2026-08-15T21:47:07.706561Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:08.585869Z","title":"React framework","venue":null,"work_id":"49609c74-c1bc-4c30-979e-e717f3e1edda","year":2013},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.710906Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:60f8c1d04d3adcafab940a0021bd22cbb3a30e8789b9eebe314209503fd42743","observation_id":"271b15e7-e93d-4fdf-a334-d587f66cd78e","resolution":{"observed_at":"2026-08-15T21:47:08.590131Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:08.571242Z","title":"Mdn web docs","venue":null,"work_id":"75674085-75fc-4875-8657-e98ffc5492d6","year":2005},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.715156Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:1012965d88cc5ceed47fea41e3c7b2b70611699df639a1152c8df10d8ffd2053","observation_id":"3a550d67-7c0d-4c16-a61d-2ce2f3e3ca1c","resolution":{"observed_at":"2026-08-15T21:47:08.576363Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2311.07599","last_updated":"2023-11-10T23:41:41Z","snapshot_observed_at":"2026-08-22T06:41:57.244863Z","submitted_at":"2023-11-10T23:41:41Z","title":"Testing LLMs on Code Generation with Varying Levels of Prompt Specificity","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.07599","snapshot_observed_at":"2026-08-15T21:47:07.719350Z","title":"Testing llms on code generation with varying levels of prompt specificity, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.719350Z"},"links":{"cited_paper":"/paper/2311.07599","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:2a623375cb84b24dd74195b840a9f35690d4c57ec77be8ca270ad4d19325123d","observation_id":"5a25c69c-92ef-484e-8d02-4068ba50ab84","resolution":{"observed_at":"2026-08-15T21:47:07.719350Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:08.557049Z","title":"Introducing swe-bench verified","venue":null,"work_id":"66709751-d91c-4c91-91e5-6a6edb4fd586","year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.723815Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:8f5688cbe243197ff9a9dabd883a9f1809952ccf13bdd9f0d1ae784398fc5a08","observation_id":"ebb585c0-1a04-4db3-b8ec-b7f23788334e","resolution":{"observed_at":"2026-08-15T21:47:08.561237Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2312.04687","last_updated":"2023-12-07T20:37:54Z","snapshot_observed_at":"2026-08-21T02:36:55.479295Z","submitted_at":"2023-12-07T20:37:54Z","title":"LLM4TDD: Best Practices for Test Driven Development Using Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.04687","snapshot_observed_at":"2026-08-15T21:47:07.728062Z","title":"and Sullivan, A","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.728062Z"},"links":{"cited_paper":"/paper/2312.04687","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:acb99178614b20045d64f835a3c33b4751b54360985461e8d6c7963eec5ab5eb","observation_id":"c5ec8b29-f0af-4db3-9c5c-a70c71903b42","resolution":{"observed_at":"2026-08-15T21:47:07.728062Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.03601","last_updated":"2024-01-07T23:01:56Z","snapshot_observed_at":"2026-08-21T00:10:56.440464Z","submitted_at":"2024-01-07T23:01:56Z","title":"InFoBench: Evaluating Instruction Following Ability in Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.03601","snapshot_observed_at":"2026-08-15T21:47:07.732870Z","title":"Infobench: Evaluating instruction following ability in large language models, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.732870Z"},"links":{"cited_paper":"/paper/2401.03601","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:a444eea237062645fa26a9d807bdd8c3db9891ff73810a7ffea7f42e8751c90e","observation_id":"cc88c425-18c2-4da5-b1dd-5dbbe5c68ea1","resolution":{"observed_at":"2026-08-15T21:47:07.732870Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1712.01815","last_updated":"2017-12-05T18:45:38Z","snapshot_observed_at":"2026-08-14T20:06:37.819179Z","submitted_at":"2017-12-05T18:45:38Z","title":"Mastering Chess and Shogi by Self-Play with a General Reinforcement Learning Algorithm","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1712.01815","snapshot_observed_at":"2026-08-15T21:47:07.737318Z","title":"Mastering chess and shogi by self-play with a general reinforcement learning algorithm, 2017","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.737318Z"},"links":{"cited_paper":"/paper/1712.01815","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:8f8c3ee712168c45b321d80f8da87a2d4d6d14d8d679098065c629d2e8c8c91c","observation_id":"1d6b7c9b-327d-494d-8a9f-0cb3537a75e6","resolution":{"observed_at":"2026-08-15T21:47:07.737318Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.07129","last_updated":"2024-04-10T16:07:38Z","snapshot_observed_at":"2026-08-16T14:02:01.042436Z","submitted_at":"2024-04-10T16:07:38Z","title":"What needs to go right for an induction head? A mechanistic study of in-context learning circuits and their formation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.07129","snapshot_observed_at":"2026-08-15T21:47:07.742026Z","title":"K., Moskovitz, T., Hill, F., Chan, S","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.742026Z"},"links":{"cited_paper":"/paper/2404.07129","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:0cd84b4b8434a36cc5663a606b0b92a57f033b7d2e2fdb7f53a0b19fbc05cb6b","observation_id":"eb5ffa12-f980-4474-b14d-ee50c8e0538a","resolution":{"observed_at":"2026-08-15T21:47:07.742026Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.11597","last_updated":"2024-06-06T12:55:17Z","snapshot_observed_at":"2026-08-16T14:17:35.885350Z","submitted_at":"2024-02-18T14:25:19Z","title":"Multi-Task Inference: Can Large Language Models Follow Multiple Instructions at Once?","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.11597","snapshot_observed_at":"2026-08-15T21:47:07.746574Z","title":"Multi-task inference: Can large language models follow multiple instructions at once?, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.746574Z"},"links":{"cited_paper":"/paper/2402.11597","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:963ab2ed8eec73ec841e72908e53eb2a4f43a5fc3fd05eeacaee4a59b2914584","observation_id":"247971a5-568a-418d-99b1-f7ce0686afbf","resolution":{"observed_at":"2026-08-15T21:47:07.746574Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.02368","last_updated":"2025-01-06T16:41:09Z","snapshot_observed_at":"2026-08-18T11:15:40.262121Z","submitted_at":"2023-10-03T18:48:31Z","title":"Reinforcement Learning from Automatic Feedback for High-Quality Unit Test Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.02368","snapshot_observed_at":"2026-08-15T21:47:07.751262Z","title":"Reinforcement learning from automatic feedback for high-quality unit test generation, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.751262Z"},"links":{"cited_paper":"/paper/2310.02368","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:485ed8da0a4316dcd8e29124e7275ba90ebaa565cccc94db014d8f3210bcb452","observation_id":"69f1cd5d-4d97-47f8-bf9a-4bac9cd57cc5","resolution":{"observed_at":"2026-08-15T21:47:07.751262Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:08.543739Z","title":"Hypothesis search: Inductive reasoning with language models","venue":null,"work_id":"3e6f966b-b0b9-422a-a2a6-47fd227ac291","year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":40,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.755863Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:ce36103134a54f80e12dc058d67d93c2ae1e9ddc87cb5bdd9a6a827c30187abd","observation_id":"d573ca1f-aeb5-4510-b5e1-5736206e424c","resolution":{"observed_at":"2026-08-15T21:47:08.548163Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2212.10560","last_updated":"2023-05-25T23:50:07Z","snapshot_observed_at":"2026-08-19T16:38:51.329886Z","submitted_at":"2022-12-20T18:59:19Z","title":"Self-Instruct: Aligning Language Models with Self-Generated Instructions","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2212.10560","snapshot_observed_at":"2026-08-15T21:47:07.760111Z","title":"A., Khashabi, D., and Hajishirzi, H","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":41,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.760111Z"},"links":{"cited_paper":"/paper/2212.10560","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:f297385eb091ac358bc7391d7f3f610bc844977be6c61edcadb6b1855f8ca932","observation_id":"19ed2752-1f40-477b-9d16-e94153571148","resolution":{"observed_at":"2026-08-15T21:47:07.760111Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2109.01652","last_updated":"2022-02-08T20:26:45Z","snapshot_observed_at":"2026-08-14T06:11:14.515796Z","submitted_at":"2021-09-03T17:55:52Z","title":"Finetuned Language Models Are Zero-Shot Learners","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2109.01652","snapshot_observed_at":"2026-08-15T21:47:07.764609Z","title":"Y., Guu, K., Yu, A","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":42,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.764609Z"},"links":{"cited_paper":"/paper/2109.01652","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:9e8f44b08c073926a548f2dd042f30bfe941ec8e013a688c2ad257e414c37aa6","observation_id":"26dfcfc7-7d51-424f-beb6-92c6198698e4","resolution":{"observed_at":"2026-08-15T21:47:07.764609Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2111.02080","last_updated":"2022-07-21T07:44:13Z","snapshot_observed_at":"2026-08-10T22:45:29.185791Z","submitted_at":"2021-11-03T09:12:33Z","title":"An Explanation of In-context Learning as Implicit Bayesian Inference","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2111.02080","snapshot_observed_at":"2026-08-15T21:47:07.769478Z","title":"M., Raghunathan, A., Liang, P., and Ma, T","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":43,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.769478Z"},"links":{"cited_paper":"/paper/2111.02080","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:d86600a3bcf7080157090e941882bcf2c90bbbb5f95eed9a56b3c0c3909c5039","observation_id":"1636f878-e7bc-42ae-b828-47a5a3305420","resolution":{"observed_at":"2026-08-15T21:47:07.769478Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:08.530093Z","title":"C.-J., Zhang, T., Patil, S","venue":null,"work_id":"6d61bbce-23b4-4699-a0e2-e74cdc3b983e","year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":44,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.774453Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:80eaa6c50f33e27301097abd318473a0d3d996ed4bfb13e7aeee60af01c6b0d7","observation_id":"0a8ee398-e969-410e-a2c8-3c5ff1b3792f","resolution":{"observed_at":"2026-08-15T21:47:08.534301Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:08.515414Z","title":"Codereval: A benchmark of pragmatic code generation with generative pre-trained models, 2023","venue":null,"work_id":"46480f86-6c02-4a5c-91ca-b089d0185ba4","year":2023},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":45,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.779593Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:b5050ca92da211ab4a549d48d62833e69cb27b6535132e17ebb2f29aa403549a","observation_id":"a124e88e-1682-4684-a24d-b62c2944c9b1","resolution":{"observed_at":"2026-08-15T21:47:08.520633Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:08.500202Z","title":null,"venue":null,"work_id":"c5f55eef-9a8e-49cf-9ec1-88f06f88e0a6","year":2022},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":46,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.784588Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:40e801d56925454bb249f6ec6297e74f25fe18604d483879c98c2d33fc352a93","observation_id":"e55de092-178a-497a-9eb0-142bbdfa4e10","resolution":{"observed_at":"2026-08-15T21:47:08.505637Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-23T06:30:58.430688+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.09629","last_updated":"2024-03-18T07:56:48Z","snapshot_observed_at":"2026-08-18T15:25:16.958531Z","submitted_at":"2024-03-14T17:58:16Z","title":"Quiet-STaR: Language Models Can Teach Themselves to Think Before Speaking","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.09629","snapshot_observed_at":"2026-08-15T21:47:07.788740Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":47,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.788740Z"},"links":{"cited_paper":"/paper/2403.09629","citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:f170b76c98cd4f3ff472f101c30dbce918722eae6913d0590054d4daceafa271","observation_id":"15bc3cae-9ad1-4898-a751-1b50d544cbeb","resolution":{"observed_at":"2026-08-15T21:47:07.788740Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:07.793104Z","title":"A survey on self-play methods in reinforcement learning, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":48,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.793104Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:d7ebf510c6c730f89bb46070abff60ac104c20f13085caa3b05fc483c9300255","observation_id":"b29a5d1f-77b3-4244-9464-5dc0aa06eeb8","resolution":{"observed_at":"2026-08-15T21:47:07.793104Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:07.798142Z","title":"write newline","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":49,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.798142Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:15ab2cb1de6490fe6856ec506f99e0bfbbf4b2127d4b72e61c378685c202ee9c","observation_id":"4d27b172-205f-4406-bbb0-0f5b22181f5a","resolution":{"observed_at":"2026-08-15T21:47:07.798142Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:07.804378Z","title":"@esa (Ref","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":50,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.804378Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:003568100a9eb70d3b2c9d9b45f071df6fe5ab18d69274deb3caffd51b3d3489","observation_id":"234bdd77-f35f-4f1d-ac78-5dfb1b12e841","resolution":{"observed_at":"2026-08-15T21:47:07.804378Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:07.809539Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":51,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.809539Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:34193a5d05b3841fd7463bd68248768f947d2799c7b9b5746a4da949777b1aae","observation_id":"906089b1-26ce-4101-9caa-fd1507ce5f58","resolution":{"observed_at":"2026-08-15T21:47:07.809539Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:47:07.814079Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation","version":1},"reference_index":52,"source":"arxiv_source","source_observed_at":"2026-08-15T21:47:07.814079Z"},"links":{"citing_paper":"/paper/2505.09027"},"observation_digest":"sha256:669cbc36f09cef9249819a30407a6107169342d5067d9e0768cf5f1338bdbe85","observation_id":"009752c6-dfa6-4d46-8b19-e93ff9b46d5e","resolution":{"observed_at":"2026-08-15T21:47:07.814079Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2505.09027","last_updated":"2025-05-13T23:47:12Z","latest_version":1,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-18T12:54:34.552706Z","submitted_at":"2025-05-13T23:47:12Z","title":"Tests as Prompt: A Test-Driven-Development Benchmark for LLM Code Generation"},"reference_resolution":{"displayed":52,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":39,"verified_exact":0,"verified_fuzzy":13},"total_outbound_references":52},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-23T06:30:58.430688+00:00","source":"crossref"},{"observed_at":"2026-08-23T06:30:53.778098+00:00","source":"retraction_watch"}],"thesis":"As of 23 August 2026, this Paper Citation Record lists 52 of 52 outbound references and 3 inbound Pith citation observations for arXiv:2505.09027."}