{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:KZDVXHMN6OXBF3ESRTXYRQTNPG","short_pith_number":"pith:KZDVXHMN","canonical_record":{"source":{"id":"2401.03065","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-01-05T20:53:51Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"1700e7185c043a55a5a1a9139892df67d8d35d960960ccb5cd1594d2edcc500b","abstract_canon_sha256":"4c190e234dc0fa17a55b121a20712b031c4c0e5b120b7649118471d954ff9ab4"},"schema_version":"1.0"},"canonical_sha256":"56475b9d8df3ae12ec928cef88c26d79b2c031c404fb081e55ef01c218e157f5","source":{"kind":"arxiv","id":"2401.03065","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2401.03065","created_at":"2026-05-17T23:39:21Z"},{"alias_kind":"arxiv_version","alias_value":"2401.03065v1","created_at":"2026-05-17T23:39:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.03065","created_at":"2026-05-17T23:39:21Z"},{"alias_kind":"pith_short_12","alias_value":"KZDVXHMN6OXB","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_16","alias_value":"KZDVXHMN6OXBF3ES","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_8","alias_value":"KZDVXHMN","created_at":"2026-05-18T12:33:37Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:KZDVXHMN6OXBF3ESRTXYRQTNPG","target":"record","payload":{"canonical_record":{"source":{"id":"2401.03065","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-01-05T20:53:51Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"1700e7185c043a55a5a1a9139892df67d8d35d960960ccb5cd1594d2edcc500b","abstract_canon_sha256":"4c190e234dc0fa17a55b121a20712b031c4c0e5b120b7649118471d954ff9ab4"},"schema_version":"1.0"},"canonical_sha256":"56475b9d8df3ae12ec928cef88c26d79b2c031c404fb081e55ef01c218e157f5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:39:21.764527Z","signature_b64":"ORuq58uChyW2tOZBhUqpYbpxgtNmGjTMDtTyAyQSP0aFD3bTFON6W9WmNYhwJ7SpASGxcRThu4Jpk/P+Vl3sAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"56475b9d8df3ae12ec928cef88c26d79b2c031c404fb081e55ef01c218e157f5","last_reissued_at":"2026-05-17T23:39:21.763738Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:39:21.763738Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2401.03065","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:39:21Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7aVBJvq0TrLkw+pw/8NMhbO2/uSwHcWtK0Kmtn+l4rapck4M9RAYp/r/96NQI3TIkM9pM/ic8Yko64u+U0FFDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-04T08:45:32.890389Z"},"content_sha256":"ec2a2ab0390b1a85bc2ad0a25b75584b7c97e89f72dbd9040910aeedb4b77589","schema_version":"1.0","event_id":"sha256:ec2a2ab0390b1a85bc2ad0a25b75584b7c97e89f72dbd9040910aeedb4b77589"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:KZDVXHMN6OXBF3ESRTXYRQTNPG","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"CRUXEval: A Benchmark for Code Reasoning, Understanding and Execution","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"CRUXEval benchmark reveals GPT-4 with chain-of-thought reaches only 75% and 81% on input and output prediction for short Python functions.","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.SE","authors_text":"Alex Gu, Armando Solar-Lezama, Baptiste Rozi\\`ere, Gabriel Synnaeve, Hugh Leather, Sida I. Wang","submitted_at":"2024-01-05T20:53:51Z","abstract_excerpt":"We present CRUXEval (Code Reasoning, Understanding, and eXecution Evaluation), a benchmark consisting of 800 Python functions (3-13 lines). Each function comes with an input-output pair, leading to two natural tasks: input prediction and output prediction. First, we propose a generic recipe for generating our execution benchmark which can be used to create future variation of the benchmark. Second, we evaluate twenty code models on our benchmark and discover that many recent high-scoring models on HumanEval do not show the same improvements on our benchmark. Third, we show that simple CoT and "},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"The best setup, GPT-4 with chain of thought (CoT), achieves a pass@1 of 75% and 81% on input and output prediction, respectively. In contrast, Code Llama 34B achieves a pass@1 of 50% and 46% on input and output prediction.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That the 800 generated functions and their input-output pairs sufficiently represent the range of code reasoning and execution challenges encountered in practical programming.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"CRUXEval benchmark shows current code models including GPT-4 achieve at most 81% on input and output prediction for short Python functions, exposing gaps not captured by HumanEval.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"CRUXEval benchmark reveals GPT-4 with chain-of-thought reaches only 75% and 81% on input and output prediction for short Python functions.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"bb1b6de01df9b6b90282bff110c4970e26dc6aa03e2ecfa9841da8aba52b3735"},"source":{"id":"2401.03065","kind":"arxiv","version":1},"verdict":{"id":"09e88c37-4f05-4deb-b894-f234e1042fde","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-14T20:53:05.420039Z","strongest_claim":"The best setup, GPT-4 with chain of thought (CoT), achieves a pass@1 of 75% and 81% on input and output prediction, respectively. In contrast, Code Llama 34B achieves a pass@1 of 50% and 46% on input and output prediction.","one_line_summary":"CRUXEval benchmark shows current code models including GPT-4 achieve at most 81% on input and output prediction for short Python functions, exposing gaps not captured by HumanEval.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That the 800 generated functions and their input-output pairs sufficiently represent the range of code reasoning and execution challenges encountered in practical programming.","pith_extraction_headline":"CRUXEval benchmark reveals GPT-4 with chain-of-thought reaches only 75% and 81% on input and output prediction for short Python functions."},"references":{"count":26,"sample":[{"doi":"","year":2017,"title":"A parallel corpus of Python functions and documentation strings for automated code documentation and code generation","work_id":"3484b4d3-268c-4c6a-8eb5-315f36c3de9a","ref_index":1,"cited_arxiv_id":"1707.02275","is_internal_anchor":true},{"doi":"","year":2023,"title":"Rishi Hazra, Gabriele Venturato, Pedro Zuidberg Dos Martires, and Luc De Raedt","work_id":"f145d48d-6da0-4a3a-945b-dc1551f01aff","ref_index":2,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":1909,"title":"CodeSearchNet Challenge: Evaluating the State of Semantic Code Search","work_id":"c800e7c9-4d30-4cea-a30c-2a54a64f9229","ref_index":3,"cited_arxiv_id":"1909.09436","is_internal_anchor":true},{"doi":"","year":2020,"title":"arXiv preprint arXiv:2305.05383 , year=","work_id":"613a2c7a-bd4f-49f6-b523-a324f88dbcbd","ref_index":4,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2022,"title":"arXiv preprint arXiv:2401.00595 , year =","work_id":"9571587f-84df-407b-baa8-6c9557faa0c7","ref_index":5,"cited_arxiv_id":"","is_internal_anchor":false}],"resolved_work":26,"snapshot_sha256":"124f6b556f9c828507a7b1f21001e1d271f7424db06bc7ee58628e01f61a3dd8","internal_anchors":2},"formal_canon":{"evidence_count":1,"snapshot_sha256":"2eb634cffbe069b3d2e2bcf1fd1717642378db482061e29b9b55f06504f0d28e"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"09e88c37-4f05-4deb-b894-f234e1042fde"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:39:21Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"yk4b3UMB4CBa3zw37L0YqDhMDG02sQNOA0a55y5TxUSgOUB1p7QqT+WeKtzz3tkRWOPneDSRVhoZ/MT5eunIAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-04T08:45:32.890937Z"},"content_sha256":"f037f76a1ed32c225317eda3ee81f0a467be5227b45fe4ffe42dd5128dc217ee","schema_version":"1.0","event_id":"sha256:f037f76a1ed32c225317eda3ee81f0a467be5227b45fe4ffe42dd5128dc217ee"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/KZDVXHMN6OXBF3ESRTXYRQTNPG/bundle.json","state_url":"https://pith.science/pith/KZDVXHMN6OXBF3ESRTXYRQTNPG/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/KZDVXHMN6OXBF3ESRTXYRQTNPG/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-04T08:45:32Z","links":{"resolver":"https://pith.science/pith/KZDVXHMN6OXBF3ESRTXYRQTNPG","bundle":"https://pith.science/pith/KZDVXHMN6OXBF3ESRTXYRQTNPG/bundle.json","state":"https://pith.science/pith/KZDVXHMN6OXBF3ESRTXYRQTNPG/state.json","well_known_bundle":"https://pith.science/.well-known/pith/KZDVXHMN6OXBF3ESRTXYRQTNPG/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:KZDVXHMN6OXBF3ESRTXYRQTNPG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"4c190e234dc0fa17a55b121a20712b031c4c0e5b120b7649118471d954ff9ab4","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-01-05T20:53:51Z","title_canon_sha256":"1700e7185c043a55a5a1a9139892df67d8d35d960960ccb5cd1594d2edcc500b"},"schema_version":"1.0","source":{"id":"2401.03065","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2401.03065","created_at":"2026-05-17T23:39:21Z"},{"alias_kind":"arxiv_version","alias_value":"2401.03065v1","created_at":"2026-05-17T23:39:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.03065","created_at":"2026-05-17T23:39:21Z"},{"alias_kind":"pith_short_12","alias_value":"KZDVXHMN6OXB","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_16","alias_value":"KZDVXHMN6OXBF3ES","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_8","alias_value":"KZDVXHMN","created_at":"2026-05-18T12:33:37Z"}],"graph_snapshots":[{"event_id":"sha256:f037f76a1ed32c225317eda3ee81f0a467be5227b45fe4ffe42dd5128dc217ee","target":"graph","created_at":"2026-05-17T23:39:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"The best setup, GPT-4 with chain of thought (CoT), achieves a pass@1 of 75% and 81% on input and output prediction, respectively. In contrast, Code Llama 34B achieves a pass@1 of 50% and 46% on input and output prediction."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That the 800 generated functions and their input-output pairs sufficiently represent the range of code reasoning and execution challenges encountered in practical programming."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"CRUXEval benchmark shows current code models including GPT-4 achieve at most 81% on input and output prediction for short Python functions, exposing gaps not captured by HumanEval."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"CRUXEval benchmark reveals GPT-4 with chain-of-thought reaches only 75% and 81% on input and output prediction for short Python functions."}],"snapshot_sha256":"bb1b6de01df9b6b90282bff110c4970e26dc6aa03e2ecfa9841da8aba52b3735"},"formal_canon":{"evidence_count":1,"snapshot_sha256":"2eb634cffbe069b3d2e2bcf1fd1717642378db482061e29b9b55f06504f0d28e"},"paper":{"abstract_excerpt":"We present CRUXEval (Code Reasoning, Understanding, and eXecution Evaluation), a benchmark consisting of 800 Python functions (3-13 lines). Each function comes with an input-output pair, leading to two natural tasks: input prediction and output prediction. First, we propose a generic recipe for generating our execution benchmark which can be used to create future variation of the benchmark. Second, we evaluate twenty code models on our benchmark and discover that many recent high-scoring models on HumanEval do not show the same improvements on our benchmark. Third, we show that simple CoT and ","authors_text":"Alex Gu, Armando Solar-Lezama, Baptiste Rozi\\`ere, Gabriel Synnaeve, Hugh Leather, Sida I. Wang","cross_cats":["cs.AI","cs.LG"],"headline":"CRUXEval benchmark reveals GPT-4 with chain-of-thought reaches only 75% and 81% on input and output prediction for short Python functions.","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-01-05T20:53:51Z","title":"CRUXEval: A Benchmark for Code Reasoning, Understanding and Execution"},"references":{"count":26,"internal_anchors":2,"resolved_work":26,"sample":[{"cited_arxiv_id":"1707.02275","doi":"","is_internal_anchor":true,"ref_index":1,"title":"A parallel corpus of Python functions and documentation strings for automated code documentation and code generation","work_id":"3484b4d3-268c-4c6a-8eb5-315f36c3de9a","year":2017},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":2,"title":"Rishi Hazra, Gabriele Venturato, Pedro Zuidberg Dos Martires, and Luc De Raedt","work_id":"f145d48d-6da0-4a3a-945b-dc1551f01aff","year":2023},{"cited_arxiv_id":"1909.09436","doi":"","is_internal_anchor":true,"ref_index":3,"title":"CodeSearchNet Challenge: Evaluating the State of Semantic Code Search","work_id":"c800e7c9-4d30-4cea-a30c-2a54a64f9229","year":1909},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":4,"title":"arXiv preprint arXiv:2305.05383 , year=","work_id":"613a2c7a-bd4f-49f6-b523-a324f88dbcbd","year":2020},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":5,"title":"arXiv preprint arXiv:2401.00595 , year =","work_id":"9571587f-84df-407b-baa8-6c9557faa0c7","year":2022}],"snapshot_sha256":"124f6b556f9c828507a7b1f21001e1d271f7424db06bc7ee58628e01f61a3dd8"},"source":{"id":"2401.03065","kind":"arxiv","version":1},"verdict":{"created_at":"2026-05-14T20:53:05.420039Z","id":"09e88c37-4f05-4deb-b894-f234e1042fde","model_set":{"reader":"grok-4.3"},"one_line_summary":"CRUXEval benchmark shows current code models including GPT-4 achieve at most 81% on input and output prediction for short Python functions, exposing gaps not captured by HumanEval.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"CRUXEval benchmark reveals GPT-4 with chain-of-thought reaches only 75% and 81% on input and output prediction for short Python functions.","strongest_claim":"The best setup, GPT-4 with chain of thought (CoT), achieves a pass@1 of 75% and 81% on input and output prediction, respectively. In contrast, Code Llama 34B achieves a pass@1 of 50% and 46% on input and output prediction.","weakest_assumption":"That the 800 generated functions and their input-output pairs sufficiently represent the range of code reasoning and execution challenges encountered in practical programming."}},"verdict_id":"09e88c37-4f05-4deb-b894-f234e1042fde"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:ec2a2ab0390b1a85bc2ad0a25b75584b7c97e89f72dbd9040910aeedb4b77589","target":"record","created_at":"2026-05-17T23:39:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"4c190e234dc0fa17a55b121a20712b031c4c0e5b120b7649118471d954ff9ab4","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-01-05T20:53:51Z","title_canon_sha256":"1700e7185c043a55a5a1a9139892df67d8d35d960960ccb5cd1594d2edcc500b"},"schema_version":"1.0","source":{"id":"2401.03065","kind":"arxiv","version":1}},"canonical_sha256":"56475b9d8df3ae12ec928cef88c26d79b2c031c404fb081e55ef01c218e157f5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"56475b9d8df3ae12ec928cef88c26d79b2c031c404fb081e55ef01c218e157f5","first_computed_at":"2026-05-17T23:39:21.763738Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:39:21.763738Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"ORuq58uChyW2tOZBhUqpYbpxgtNmGjTMDtTyAyQSP0aFD3bTFON6W9WmNYhwJ7SpASGxcRThu4Jpk/P+Vl3sAw==","signature_status":"signed_v1","signed_at":"2026-05-17T23:39:21.764527Z","signed_message":"canonical_sha256_bytes"},"source_id":"2401.03065","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:ec2a2ab0390b1a85bc2ad0a25b75584b7c97e89f72dbd9040910aeedb4b77589","sha256:f037f76a1ed32c225317eda3ee81f0a467be5227b45fe4ffe42dd5128dc217ee"],"state_sha256":"7f54b55a8753ec926bc664f2271dd529805e5b0589e5b7fd083e79c9264e492d"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XBZVk0vMUG0+UPeq94xI/S8RRhOvOo+esZDwxTq2iPC2iCffMejaCtbqZyjB08Zyhc7LRR9XTdvyaBXrlgrMDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-04T08:45:32.893962Z","bundle_sha256":"eb37f0afdc4135f029a47874deca9bbb84c97a1b8a2d8ba9696e5b95dfcf2550"}}