{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:I7R2ORFJJS2MTPLW4MU4YAAJS7","short_pith_number":"pith:I7R2ORFJ","schema_version":"1.0","canonical_sha256":"47e3a744a94cb4c9bd76e329cc000997c78e6c16db887ef51f2d10e8cb522125","source":{"kind":"arxiv","id":"2311.09835","version":5},"attestation_state":"computed","paper":{"title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Arman Cohan, Baobao Chang, Haozhe Zhao, Helan hu, Junjie Lu, Kaikai An, Liang Chen, Mark Gerstein, Ruijun Huang, Sheng Chen, Shuzheng Si, Tianyu Liu, Wangchunshu Zhou, Xiangru Tang, Yanjun Shao, Yan Wang, Yichi Zhang, Yilun Zhao, Yin Fang, Yujia Qin, Yuliang Liu, Zefan Cai, Zexuan Deng, Zhiwei Jiang","submitted_at":"2023-11-16T12:03:21Z","abstract_excerpt":"Despite Large Language Models (LLMs) like GPT-4 achieving impressive results in function-level code generation, they struggle with repository-scale code understanding (e.g., coming up with the right arguments for calling routines), requiring a deeper comprehension of complex file interactions. Also, recently, people have developed LLM agents that attempt to interact with repository code (e.g., compiling and evaluating its execution), prompting the need to evaluate their performance. These gaps have motivated our development of ML-Bench, a benchmark rooted in real-world programming applications"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2311.09835","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-16T12:03:21Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"d852bbc0aead1ab4c321944655dab59321ac7bc3894c8148e4ac8e8c1fa35585","abstract_canon_sha256":"1445b578891a6698e4f704b2ab8c63ddb62a09ce00385ac4c007fd65516d61cf"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:57:34.750087Z","signature_b64":"qswhFVwlIULo1kozB96L52FYHrgRLddT3LjO4shu90ATqXLX/7O6NV+HcaRdIV8v6dPY2b15VKBuSsnEuUNkCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"47e3a744a94cb4c9bd76e329cc000997c78e6c16db887ef51f2d10e8cb522125","last_reissued_at":"2026-07-05T08:57:34.749564Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:57:34.749564Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Arman Cohan, Baobao Chang, Haozhe Zhao, Helan hu, Junjie Lu, Kaikai An, Liang Chen, Mark Gerstein, Ruijun Huang, Sheng Chen, Shuzheng Si, Tianyu Liu, Wangchunshu Zhou, Xiangru Tang, Yanjun Shao, Yan Wang, Yichi Zhang, Yilun Zhao, Yin Fang, Yujia Qin, Yuliang Liu, Zefan Cai, Zexuan Deng, Zhiwei Jiang","submitted_at":"2023-11-16T12:03:21Z","abstract_excerpt":"Despite Large Language Models (LLMs) like GPT-4 achieving impressive results in function-level code generation, they struggle with repository-scale code understanding (e.g., coming up with the right arguments for calling routines), requiring a deeper comprehension of complex file interactions. Also, recently, people have developed LLM agents that attempt to interact with repository code (e.g., compiling and evaluating its execution), prompting the need to evaluate their performance. These gaps have motivated our development of ML-Bench, a benchmark rooted in real-world programming applications"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.09835","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2311.09835/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2311.09835","created_at":"2026-07-05T08:57:34.749630+00:00"},{"alias_kind":"arxiv_version","alias_value":"2311.09835v5","created_at":"2026-07-05T08:57:34.749630+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.09835","created_at":"2026-07-05T08:57:34.749630+00:00"},{"alias_kind":"pith_short_12","alias_value":"I7R2ORFJJS2M","created_at":"2026-07-05T08:57:34.749630+00:00"},{"alias_kind":"pith_short_16","alias_value":"I7R2ORFJJS2MTPLW","created_at":"2026-07-05T08:57:34.749630+00:00"},{"alias_kind":"pith_short_8","alias_value":"I7R2ORFJ","created_at":"2026-07-05T08:57:34.749630+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":7,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.12376","citing_title":"ProfiliTable: Profiling-Driven Tabular Data Processing via Agentic Workflows","ref_index":36,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08678","citing_title":"MLS-Bench: A Holistic and Rigorous Assessment of AI Systems on Building Better AI","ref_index":95,"is_internal_anchor":false},{"citing_arxiv_id":"2410.07095","citing_title":"MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering","ref_index":28,"is_internal_anchor":false},{"citing_arxiv_id":"2506.04565","citing_title":"From Standalone LLMs to Integrated Intelligence: A Survey of Compound Al Systems","ref_index":173,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12376","citing_title":"ProfiliTable: Profiling-Driven Tabular Data Processing via Agentic Workflows","ref_index":34,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08678","citing_title":"MLS-Bench: A Holistic and Rigorous Assessment of AI Systems on Building Better AI","ref_index":93,"is_internal_anchor":false},{"citing_arxiv_id":"2308.00352","citing_title":"MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework","ref_index":33,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7","json":"https://pith.science/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7.json","graph_json":"https://pith.science/api/pith-number/I7R2ORFJJS2MTPLW4MU4YAAJS7/graph.json","events_json":"https://pith.science/api/pith-number/I7R2ORFJJS2MTPLW4MU4YAAJS7/events.json","paper":"https://pith.science/paper/I7R2ORFJ"},"agent_actions":{"view_html":"https://pith.science/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7","download_json":"https://pith.science/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7.json","view_paper":"https://pith.science/paper/I7R2ORFJ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2311.09835&json=true","fetch_graph":"https://pith.science/api/pith-number/I7R2ORFJJS2MTPLW4MU4YAAJS7/graph.json","fetch_events":"https://pith.science/api/pith-number/I7R2ORFJJS2MTPLW4MU4YAAJS7/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7/action/timestamp_anchor","attest_storage":"https://pith.science/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7/action/storage_attestation","attest_author":"https://pith.science/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7/action/author_attestation","sign_citation":"https://pith.science/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7/action/citation_signature","submit_replication":"https://pith.science/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7/action/replication_record"}},"created_at":"2026-07-05T08:57:34.749630+00:00","updated_at":"2026-07-05T08:57:34.749630+00:00"}