{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:KFEYUAX4YJCNY4PZO256EX6E3Z","short_pith_number":"pith:KFEYUAX4","canonical_record":{"source":{"id":"2411.15114","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-11-22T18:30:46Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"9ba1249590ebf1b0f21cec01756bae0115bd19dbb4c34db30c04131bf866b035","abstract_canon_sha256":"e415f1a30f91f588556fcf190cc61137bc95c139b5df3209131ac18c3896bb97"},"schema_version":"1.0"},"canonical_sha256":"51498a02fcc244dc71f976bbe25fc4de63ea59d8c44df3ab4f4085fba5294f8a","source":{"kind":"arxiv","id":"2411.15114","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.15114","created_at":"2026-07-05T11:10:02Z"},{"alias_kind":"arxiv_version","alias_value":"2411.15114v2","created_at":"2026-07-05T11:10:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.15114","created_at":"2026-07-05T11:10:02Z"},{"alias_kind":"pith_short_12","alias_value":"KFEYUAX4YJCN","created_at":"2026-07-05T11:10:02Z"},{"alias_kind":"pith_short_16","alias_value":"KFEYUAX4YJCNY4PZ","created_at":"2026-07-05T11:10:02Z"},{"alias_kind":"pith_short_8","alias_value":"KFEYUAX4","created_at":"2026-07-05T11:10:02Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:KFEYUAX4YJCNY4PZO256EX6E3Z","target":"record","payload":{"canonical_record":{"source":{"id":"2411.15114","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-11-22T18:30:46Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"9ba1249590ebf1b0f21cec01756bae0115bd19dbb4c34db30c04131bf866b035","abstract_canon_sha256":"e415f1a30f91f588556fcf190cc61137bc95c139b5df3209131ac18c3896bb97"},"schema_version":"1.0"},"canonical_sha256":"51498a02fcc244dc71f976bbe25fc4de63ea59d8c44df3ab4f4085fba5294f8a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:10:02.659942Z","signature_b64":"mHcL+fnGk7Pir+yrLOWNDnds+UNRrYy3hV+kkLyJdr1DC/+YrFJv7GV23s518AGMTNrDErfqhr/gCKbsQ4BZBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"51498a02fcc244dc71f976bbe25fc4de63ea59d8c44df3ab4f4085fba5294f8a","last_reissued_at":"2026-07-05T11:10:02.659439Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:10:02.659439Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2411.15114","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:10:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"G8cctYXnxa7bCo9fCaZP3KOVkDoiHxnbOwUC9hRKX5vZn/VtcfTNsL+HIs6qqCji891mRJQdwsKkJs/pdLr2Aw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T14:49:03.944798Z"},"content_sha256":"1bb76ec6759b6d069ca183af268c31250345d35d17272b60b702786903dbc8e3","schema_version":"1.0","event_id":"sha256:1bb76ec6759b6d069ca183af268c31250345d35d17272b60b702786903dbc8e3"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:KFEYUAX4YJCNY4PZO256EX6E3Z","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Aron Lajko, Ben West, Brian Goodrich, Elena Ericheva, Elizabeth Barnes, Hjalmar Wijk, Holden Karnofsky, Jai Dhyani, Joel Becker, Josh Clymer, Katharyn Garcia, Lawrence Chan, Lucas Sato, Maksym Taran, Megan Kinniment, Michael Chen, Neev Parikh, Nikola Jurkovic, Sami Jawhar, Seraphina Nix, Tao Lin, Thomas Broadley, William Saunders","submitted_at":"2024-11-22T18:30:46Z","abstract_excerpt":"Frontier AI safety policies highlight automation of AI research and development (R&D) by AI agents as an important capability to anticipate. However, there exist few evaluations for AI R&D capabilities, and none that are highly realistic and have a direct comparison to human performance. We introduce RE-Bench (Research Engineering Benchmark, v1), which consists of 7 challenging, open-ended ML research engineering environments and data from 71 8-hour attempts by 61 distinct human experts. We confirm that our experts make progress in the environments given 8 hours, with 82% of expert attempts ac"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.15114","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.15114/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:10:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"isx4aD64IbbdFPPw3WftdKUMIF0ZVnbburrpT+9Z8giPmj4T2SVsG9f3DYn4BV+GK/xK/HpCPJMXBrHdAoG8Bw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T14:49:03.945168Z"},"content_sha256":"2d2baa00aaa4c9288ae4af68fc22a504b237e55c4b746f02fb2a34566c29eea9","schema_version":"1.0","event_id":"sha256:2d2baa00aaa4c9288ae4af68fc22a504b237e55c4b746f02fb2a34566c29eea9"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/KFEYUAX4YJCNY4PZO256EX6E3Z/bundle.json","state_url":"https://pith.science/pith/KFEYUAX4YJCNY4PZO256EX6E3Z/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/KFEYUAX4YJCNY4PZO256EX6E3Z/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T14:49:03Z","links":{"resolver":"https://pith.science/pith/KFEYUAX4YJCNY4PZO256EX6E3Z","bundle":"https://pith.science/pith/KFEYUAX4YJCNY4PZO256EX6E3Z/bundle.json","state":"https://pith.science/pith/KFEYUAX4YJCNY4PZO256EX6E3Z/state.json","well_known_bundle":"https://pith.science/.well-known/pith/KFEYUAX4YJCNY4PZO256EX6E3Z/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:KFEYUAX4YJCNY4PZO256EX6E3Z","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e415f1a30f91f588556fcf190cc61137bc95c139b5df3209131ac18c3896bb97","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-11-22T18:30:46Z","title_canon_sha256":"9ba1249590ebf1b0f21cec01756bae0115bd19dbb4c34db30c04131bf866b035"},"schema_version":"1.0","source":{"id":"2411.15114","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.15114","created_at":"2026-07-05T11:10:02Z"},{"alias_kind":"arxiv_version","alias_value":"2411.15114v2","created_at":"2026-07-05T11:10:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.15114","created_at":"2026-07-05T11:10:02Z"},{"alias_kind":"pith_short_12","alias_value":"KFEYUAX4YJCN","created_at":"2026-07-05T11:10:02Z"},{"alias_kind":"pith_short_16","alias_value":"KFEYUAX4YJCNY4PZ","created_at":"2026-07-05T11:10:02Z"},{"alias_kind":"pith_short_8","alias_value":"KFEYUAX4","created_at":"2026-07-05T11:10:02Z"}],"graph_snapshots":[{"event_id":"sha256:2d2baa00aaa4c9288ae4af68fc22a504b237e55c4b746f02fb2a34566c29eea9","target":"graph","created_at":"2026-07-05T11:10:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2411.15114/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Frontier AI safety policies highlight automation of AI research and development (R&D) by AI agents as an important capability to anticipate. However, there exist few evaluations for AI R&D capabilities, and none that are highly realistic and have a direct comparison to human performance. We introduce RE-Bench (Research Engineering Benchmark, v1), which consists of 7 challenging, open-ended ML research engineering environments and data from 71 8-hour attempts by 61 distinct human experts. We confirm that our experts make progress in the environments given 8 hours, with 82% of expert attempts ac","authors_text":"Aron Lajko, Ben West, Brian Goodrich, Elena Ericheva, Elizabeth Barnes, Hjalmar Wijk, Holden Karnofsky, Jai Dhyani, Joel Becker, Josh Clymer, Katharyn Garcia, Lawrence Chan, Lucas Sato, Maksym Taran, Megan Kinniment, Michael Chen, Neev Parikh, Nikola Jurkovic, Sami Jawhar, Seraphina Nix, Tao Lin, Thomas Broadley, William Saunders","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-11-22T18:30:46Z","title":"RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.15114","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1bb76ec6759b6d069ca183af268c31250345d35d17272b60b702786903dbc8e3","target":"record","created_at":"2026-07-05T11:10:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e415f1a30f91f588556fcf190cc61137bc95c139b5df3209131ac18c3896bb97","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-11-22T18:30:46Z","title_canon_sha256":"9ba1249590ebf1b0f21cec01756bae0115bd19dbb4c34db30c04131bf866b035"},"schema_version":"1.0","source":{"id":"2411.15114","kind":"arxiv","version":2}},"canonical_sha256":"51498a02fcc244dc71f976bbe25fc4de63ea59d8c44df3ab4f4085fba5294f8a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"51498a02fcc244dc71f976bbe25fc4de63ea59d8c44df3ab4f4085fba5294f8a","first_computed_at":"2026-07-05T11:10:02.659439Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:10:02.659439Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"mHcL+fnGk7Pir+yrLOWNDnds+UNRrYy3hV+kkLyJdr1DC/+YrFJv7GV23s518AGMTNrDErfqhr/gCKbsQ4BZBg==","signature_status":"signed_v1","signed_at":"2026-07-05T11:10:02.659942Z","signed_message":"canonical_sha256_bytes"},"source_id":"2411.15114","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1bb76ec6759b6d069ca183af268c31250345d35d17272b60b702786903dbc8e3","sha256:2d2baa00aaa4c9288ae4af68fc22a504b237e55c4b746f02fb2a34566c29eea9"],"state_sha256":"a5cf32f1262c10a3c25f20081d3c06c100b19de31addf852dc8bb6160986ebed"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7TeLy9fZTUwBGiMxozOFzrB2AJo1Q+XNmajB5i5YjUWTLjJ6CtuQZD/UMV3suZlilkAcVgRP/pZGuGjo61CCCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T14:49:03.947058Z","bundle_sha256":"90282632acca8d2e298064cff4937257f803e6b005efe29d4143ee9db1cec620"}}