{"as_of":"2026-08-22T06:04:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:c9a5933d17b1003689d90de4be3091838076381256b91e2f159aac4b549b3f1b","coverage":[{"denominator":0,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":20,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":20,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-21T06:32:19.484+00:00","state":"measured"},{"denominator":20,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":20,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T20:41:08.909862Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-01T14:05:46.736275Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":"2311.09835","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-07-01T14:05:46.736275Z","title":"ML - Bench : Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository - Level Code , August 2024","venue":null,"work_id":"c1ece405-4a98-4fe9-90a5-42fa6801339d","year":2023},"citing_paper":{"arxiv_id":"2308.00352","last_updated":"2024-11-01T14:36:52Z","snapshot_observed_at":"2026-08-12T16:02:21.969968Z","submitted_at":"2023-08-01T07:49:10Z","title":"MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework","version":7},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-05-11T03:43:18.632292Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2308.00352"},"observation_digest":"sha256:4d90f33737765490f3c06d2564877103adeaa5131a01fccfdac020b5c7b5287a","observation_id":"ebf24730-2b79-4bb7-ac86-c752e9e58ab9","resolution":{"observed_at":"2026-05-11T03:43:18.933691Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":"2311.09835","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-07-01T14:05:46.736275Z","title":"ML - Bench : Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository - Level Code , August 2024","venue":null,"work_id":"c1ece405-4a98-4fe9-90a5-42fa6801339d","year":2023},"citing_paper":{"arxiv_id":"2410.07095","last_updated":"2025-02-26T11:57:30Z","snapshot_observed_at":"2026-08-12T16:49:10.970507Z","submitted_at":"2024-10-09T17:34:27Z","title":"MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering","version":6},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-05-23T19:11:20.600633Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2410.07095"},"observation_digest":"sha256:7be8cc15d0ba423cc918b20da07ff03e22b48e56f4bfa03b959d917f62d1d637","observation_id":"230b2e79-42ba-4507-bf2f-bb3b8e302bd2","resolution":{"observed_at":"2026-05-23T19:13:21.535616Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-08-12T19:02:19.266981Z","title":"ML-Bench : Evaluating large language models and agents for machine learning tasks on repository-level code","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.11910","last_updated":"2024-11-24T12:59:44Z","snapshot_observed_at":"2026-08-18T03:56:12.036803Z","submitted_at":"2024-11-17T13:40:35Z","title":"AIGS: Generating Science from AI-Powered Automated Falsification","version":2},"reference_index":54,"source":"arxiv_source","source_observed_at":"2026-08-12T19:02:19.266981Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2411.11910"},"observation_digest":"sha256:a81e73e1ba1cfdafac3707a4b32319a1ae50cf5583e85474d8cb537e7b2781ce","observation_id":"a4df26b6-6cc6-4622-9203-23ed6fd64527","resolution":{"observed_at":"2026-08-12T19:02:19.266981Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-08-15T20:41:08.909862Z","title":"Ml-bench: Evaluating large language models and agents for machine learning tasks on repository-level code.arXiv preprint arXiv:2311.09835, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.12331","last_updated":"2025-05-20T02:24:56Z","snapshot_observed_at":"2026-08-19T01:04:33.324911Z","submitted_at":"2025-05-18T09:53:51Z","title":"OSS-Bench: Benchmark Generator for Coding LLMs","version":2},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-15T20:41:08.909862Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2505.12331"},"observation_digest":"sha256:723ff996bfb6f40e90cd5c5fab44f0d584fd56a4bf687d4f3bd947927876adaf","observation_id":"19ebd076-1d70-427d-912f-77193fece6a3","resolution":{"observed_at":"2026-08-15T20:41:08.909862Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-08-15T20:18:59.955968Z","title":"ML-Bench: Evaluating large language models and agents for machine learning tasks on repository-level code","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.13291","last_updated":"2025-05-19T16:11:23Z","snapshot_observed_at":"2026-08-15T20:13:53.875819Z","submitted_at":"2025-05-19T16:11:23Z","title":"TimeSeriesGym: A Scalable Benchmark for (Time Series) Machine Learning Engineering Agents","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-15T20:18:59.955968Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2505.13291"},"observation_digest":"sha256:fb50ce6232be8103153e1a9883c4d19ef76871e036e8d2a4e720b49f9b7c199e","observation_id":"ebfff6ab-3eb5-467d-97f9-cefbfd68fe81","resolution":{"observed_at":"2026-08-15T20:18:59.955968Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-08-07T13:53:16.268485Z","title":"Ml-bench: Evaluating large language models and agents for machine learning tasks on repository-level code","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.21577","last_updated":"2025-08-25T13:40:36Z","snapshot_observed_at":"2026-08-22T00:52:45.113066Z","submitted_at":"2025-05-27T08:35:05Z","title":"RepoMaster: Autonomous Exploration and Understanding of GitHub Repositories for Complex Task Solving","version":3},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-07T13:53:16.268485Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2505.21577"},"observation_digest":"sha256:93890e1baaf9410b82c9b68e6ee97b805e447c8a89d429a871d7fa3389c5ce77","observation_id":"3308b1be-fe72-42e4-a9a4-f700438c81a8","resolution":{"observed_at":"2026-08-07T13:53:16.268485Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":"2311.09835","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-07-01T14:05:46.736275Z","title":"ML - Bench : Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository - Level Code , August 2024","venue":null,"work_id":"c1ece405-4a98-4fe9-90a5-42fa6801339d","year":2023},"citing_paper":{"arxiv_id":"2506.04565","last_updated":"2026-05-08T15:50:43Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-06-05T02:34:43Z","title":"From Standalone LLMs to Integrated Intelligence: A Survey of Compound Al Systems","version":2},"reference_index":173,"source":"pdf_text","source_observed_at":"2026-05-19T11:49:36.574471Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2506.04565"},"observation_digest":"sha256:f97c3c222e9edd817d8e31b48127fd712dc9581c494eeac4aeeb29e3e0cedb69","observation_id":"0a3c29a7-66ad-42f9-8b46-abb6bc0485fa","resolution":{"observed_at":"2026-05-19T11:52:16.430624Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-08-07T06:00:18.492754Z","title":"Ml-bench: Evaluating large lan- guage models and agents for machine learning tasks on repository-level code,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:18.492754Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:c06f448bc1783c88529d0ac739abc93e9300e8b5b80d08368a740666827c454e","observation_id":"d59ecd38-95a2-4267-810e-79a9d494d2f9","resolution":{"observed_at":"2026-08-07T06:00:18.492754Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-08-06T23:51:19.113797Z","title":"InThe Twelfth In- ternational Conference on Learning Representations, ICLR 2024, Vienna, Austria, May 7-11, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17335","last_updated":"2025-06-19T07:04:16Z","snapshot_observed_at":"2026-08-20T22:44:22.366302Z","submitted_at":"2025-06-19T07:04:16Z","title":"LMR-BENCH: Evaluating LLM Agent's Ability on Reproducing Language Modeling Research","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-06T23:51:19.113797Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2506.17335"},"observation_digest":"sha256:d6b2a9315117e86751794228d15366ee33e6181c076b17d57441521791fef747","observation_id":"883b6106-d5c8-4d33-95d9-44f1c38e3cb9","resolution":{"observed_at":"2026-08-06T23:51:19.113797Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-08-06T23:26:56.893012Z","title":"Ml-bench: Evaluating large language models and agents for machine learning tasks on repository-level code","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18096","last_updated":"2025-09-03T15:32:23Z","snapshot_observed_at":"2026-08-15T20:12:18.032530Z","submitted_at":"2025-06-22T16:52:48Z","title":"Deep Research Agents: A Systematic Examination And Roadmap","version":2},"reference_index":105,"source":"pdf_text","source_observed_at":"2026-08-06T23:26:56.893012Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2506.18096"},"observation_digest":"sha256:89f983e31e9fefce3b51d971f57ae6cebb3afc597c7958aaf353978db1ab2dc6","observation_id":"9c8a6073-8551-4f3e-9228-a2b53f7f9e2a","resolution":{"observed_at":"2026-08-06T23:26:56.893012Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-08-06T16:30:31.039397Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.13300","last_updated":"2025-07-17T17:09:22Z","snapshot_observed_at":"2026-08-14T05:11:52.199753Z","submitted_at":"2025-07-17T17:09:22Z","title":"AbGen: Evaluating Large Language Models in Ablation Study Design and Evaluation for Scientific Research","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-06T16:30:31.039397Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2507.13300"},"observation_digest":"sha256:a86f2c06134d3605f90b291974e9001b220976e6b5806bb6e1bdbb89e238e268","observation_id":"c6f21ed6-1294-43ac-9006-709465b70578","resolution":{"observed_at":"2026-08-06T16:30:31.039397Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-08-05T12:24:02.277658Z","title":"Ml-bench: Evaluating large language models and agents for machine learning tasks on repository-level code","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2509.01684","last_updated":"2025-09-01T18:04:10Z","snapshot_observed_at":"2026-08-21T00:38:04.074355Z","submitted_at":"2025-09-01T18:04:10Z","title":"Reinforcement Learning for Machine Learning Engineering Agents","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-05T12:24:02.277658Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2509.01684"},"observation_digest":"sha256:90530ecf5ebfd5b309f252ef3991dda9f3b57fac0eeaa4eb71e7266158a56df7","observation_id":"744c909d-1d45-4ef6-aa4d-29e09ab1a074","resolution":{"observed_at":"2026-08-05T12:24:02.277658Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-08-02T23:31:16.475615Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2602.13723","last_updated":"2026-08-09T07:41:10Z","snapshot_observed_at":"2026-08-16T12:39:33.590503Z","submitted_at":"2026-02-14T11:07:58Z","title":"Compiling Large Multi-Modal Requirement Documents into Runnable Software Systems: From an Agentic Test-Driven Perspective","version":5},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-02T23:31:16.475615Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2602.13723"},"observation_digest":"sha256:39769bf384416a950a8a667241474108abdb845c02d8c2490ab2593455f31182","observation_id":"174c2508-feb8-4a56-86a8-86a342144568","resolution":{"observed_at":"2026-08-02T23:31:16.475615Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":"2311.09835","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-07-01T14:05:46.736275Z","title":"ML - Bench : Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository - Level Code , August 2024","venue":null,"work_id":"c1ece405-4a98-4fe9-90a5-42fa6801339d","year":2023},"citing_paper":{"arxiv_id":"2605.08678","last_updated":"2026-07-06T13:36:13Z","snapshot_observed_at":"2026-08-14T16:53:14.701034Z","submitted_at":"2026-05-09T04:29:46Z","title":"MLS-Bench: A Holistic and Rigorous Assessment of AI Systems on Building Better AI","version":1},"reference_index":93,"source":"pdf_text","source_observed_at":"2026-05-12T01:13:35.990078Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2605.08678"},"observation_digest":"sha256:f642eaff9b5c43106359829e565563fe9e471708f0f7fe86bda0709b41423143","observation_id":"18816a07-66c5-4ba8-a943-1a7a84a17885","resolution":{"observed_at":"2026-05-12T08:21:25.084754Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":"2311.09835","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-07-01T14:05:46.736275Z","title":"ML - Bench : Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository - Level Code , August 2024","venue":null,"work_id":"c1ece405-4a98-4fe9-90a5-42fa6801339d","year":2023},"citing_paper":{"arxiv_id":"2605.08678","last_updated":"2026-07-06T13:36:13Z","snapshot_observed_at":"2026-08-14T16:53:14.701034Z","submitted_at":"2026-05-09T04:29:46Z","title":"MLS-Bench: A Holistic and Rigorous Assessment of AI Systems on Building Better AI","version":2},"reference_index":95,"source":"pdf_text","source_observed_at":"2026-06-30T23:12:57.154537Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2605.08678"},"observation_digest":"sha256:222625962fc9d87c71c39e3273106c7d235c625e841a51c4109f96a8d7bf5f63","observation_id":"cecd6380-a041-4259-aa06-cedd57d3e95b","resolution":{"observed_at":"2026-07-01T13:25:46.005528Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-07-12T17:14:49.310598Z","title":"Ml-bench: Evaluating large language models and agents for machine learning tasks on repository-level code.arXiv preprint arXiv:2311.09835, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2605.08678","last_updated":"2026-07-06T13:36:13Z","snapshot_observed_at":"2026-08-14T16:53:14.701034Z","submitted_at":"2026-05-09T04:29:46Z","title":"MLS-Bench: A Holistic and Rigorous Assessment of AI Systems on Building Better AI","version":3},"reference_index":94,"source":"pdf_text","source_observed_at":"2026-07-12T17:14:49.310598Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2605.08678"},"observation_digest":"sha256:36a0d6af7112d50d05e5973fc85e183a85be7c23d902a6d5e15d0e50e088a21e","observation_id":"ef75ebca-0ea7-44ac-8c98-c58ed522f8cb","resolution":{"observed_at":"2026-07-12T17:14:49.310598Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":"2311.09835","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-07-01T14:05:46.736275Z","title":"ML - Bench : Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository - Level Code , August 2024","venue":null,"work_id":"c1ece405-4a98-4fe9-90a5-42fa6801339d","year":2023},"citing_paper":{"arxiv_id":"2605.12376","last_updated":"2026-06-04T17:58:18Z","snapshot_observed_at":"2026-08-15T10:05:43.294700Z","submitted_at":"2026-05-12T16:42:38Z","title":"ProfiliTable: Profiling-Driven Tabular Data Processing via Agentic Workflows","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-05-13T05:56:36.312877Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2605.12376"},"observation_digest":"sha256:20479523513b3eeb6c2427fd32505868aa3473bb63d242644acb8198bbb732c9","observation_id":"637a3010-188f-4e81-8dbc-ff3578f8ba1d","resolution":{"observed_at":"2026-05-13T05:57:22.501265Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":"2311.09835","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-07-01T14:05:46.736275Z","title":"ML - Bench : Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository - Level Code , August 2024","venue":null,"work_id":"c1ece405-4a98-4fe9-90a5-42fa6801339d","year":2023},"citing_paper":{"arxiv_id":"2605.12376","last_updated":"2026-06-04T17:58:18Z","snapshot_observed_at":"2026-08-15T10:05:43.294700Z","submitted_at":"2026-05-12T16:42:38Z","title":"ProfiliTable: Profiling-Driven Tabular Data Processing via Agentic Workflows","version":2},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-06-30T22:18:45.189576Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2605.12376"},"observation_digest":"sha256:3471f90aac34c4ba37a3754dc3cf9fe9ea88109e7fbf92197cc7738a79a1326d","observation_id":"26cc257b-1017-4935-887b-1bbe3b2806a1","resolution":{"observed_at":"2026-07-01T14:05:46.737909Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-07-31T01:39:48.754483Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.25090","last_updated":"2026-07-27T21:30:39Z","snapshot_observed_at":"2026-08-19T02:05:13.794378Z","submitted_at":"2026-07-27T21:30:39Z","title":"Matryoshka Agent: Unfolding Sub-Agents for Long-Horizon Machine Learning Engineering","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-07-31T01:39:48.754483Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2607.25090"},"observation_digest":"sha256:377f15ea8cb8ca8f9c68605063e7551fcb010bb69c54615ecfd47ccad9d83ef3","observation_id":"3e1dffac-109e-4453-9e1c-48b967604fc4","resolution":{"observed_at":"2026-07-31T01:39:48.754483Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-08-03T03:15:14.160911Z","title":"ML-Bench: Evaluating large language models and agents for machine learning tasks on repository-level code, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.29626","last_updated":"2026-07-31T16:58:00Z","snapshot_observed_at":"2026-08-14T22:04:54.419009Z","submitted_at":"2026-07-31T16:58:00Z","title":"AgentHPOBench: A Benchmark For Evaluating LLM Agents as Sequential Hyperparameter Optimizers","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-03T03:15:14.160911Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2607.29626"},"observation_digest":"sha256:8bca84fbd85d5605178ead3804ee05e01fb0e58dcd5da4fd6b1e70fb44f2d815","observation_id":"b8946c5c-24df-47d6-8697-164497746df3","resolution":{"observed_at":"2026-08-03T03:15:14.160911Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2311.09835/citation-record","integrity":"/paper/2311.09835/integrity","json":"/paper/2311.09835/citation-record.json","paper":"/paper/2311.09835"},"outbound":[],"paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","latest_version":5,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-16T14:42:42.411945Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code"},"reference_resolution":{"displayed":0,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":0,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":0},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"thesis":"As of 22 August 2026, this Paper Citation Record lists 0 of 0 outbound references and 20 inbound Pith citation observations for arXiv:2311.09835."}