{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:PQQC7F4H7VGFOO4ZGUIJFAOAYL","short_pith_number":"pith:PQQC7F4H","canonical_record":{"source":{"id":"2201.03544","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2022-01-10T18:58:52Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"095490091c403c86f8c23378791367d7633e83c21fbfa36d10be2655d5908b8e","abstract_canon_sha256":"6913c36b9ab8b86c32aae9efedbf89d259fbf5449156aa83678a4066709e0c76"},"schema_version":"1.0"},"canonical_sha256":"7c202f9787fd4c573b9935109281c0c2f54a705f54b072d7ce644bb3c15a73c0","source":{"kind":"arxiv","id":"2201.03544","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2201.03544","created_at":"2026-05-21T07:03:14Z"},{"alias_kind":"arxiv_version","alias_value":"2201.03544v2","created_at":"2026-05-21T07:03:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2201.03544","created_at":"2026-05-21T07:03:14Z"},{"alias_kind":"pith_short_12","alias_value":"PQQC7F4H7VGF","created_at":"2026-05-21T07:03:14Z"},{"alias_kind":"pith_short_16","alias_value":"PQQC7F4H7VGFOO4Z","created_at":"2026-05-21T07:03:14Z"},{"alias_kind":"pith_short_8","alias_value":"PQQC7F4H","created_at":"2026-05-21T07:03:14Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:PQQC7F4H7VGFOO4ZGUIJFAOAYL","target":"record","payload":{"canonical_record":{"source":{"id":"2201.03544","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2022-01-10T18:58:52Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"095490091c403c86f8c23378791367d7633e83c21fbfa36d10be2655d5908b8e","abstract_canon_sha256":"6913c36b9ab8b86c32aae9efedbf89d259fbf5449156aa83678a4066709e0c76"},"schema_version":"1.0"},"canonical_sha256":"7c202f9787fd4c573b9935109281c0c2f54a705f54b072d7ce644bb3c15a73c0","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-21T07:03:14.354677Z","signature_b64":"gtbrrXowFli5ge/xaNPRtC4XDe8rr3G3FxBR3hbMFKEQ5Zvi380boKnbIfp+tTqMBlvk9qonWPm+Ar0nTRHXBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7c202f9787fd4c573b9935109281c0c2f54a705f54b072d7ce644bb3c15a73c0","last_reissued_at":"2026-05-21T07:03:14.352943Z","signature_status":"signed_v1","first_computed_at":"2026-05-21T07:03:14.352943Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2201.03544","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-21T07:03:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rw451Oc4qurxp/JGqcMaD0Vai5jRvoR2VizUB/GBW48bhZAE5cC4vqvRHe0E55hct9Iw2CXyJdyBbl02M9BvAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-21T11:45:23.301904Z"},"content_sha256":"78f6566c4aab50372360e924dc1aba304499cb079752da32ac9e35c7df349e77","schema_version":"1.0","event_id":"sha256:78f6566c4aab50372360e924dc1aba304499cb079752da32ac9e35c7df349e77"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:PQQC7F4H7VGFOO4ZGUIJFAOAYL","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"The Effects of Reward Misspecification: Mapping and Mitigating Misaligned Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Alexander Pan, Jacob Steinhardt, Kush Bhatia","submitted_at":"2022-01-10T18:58:52Z","abstract_excerpt":"Reward hacking -- where RL agents exploit gaps in misspecified reward functions -- has been widely observed, but not yet systematically studied. To understand how reward hacking arises, we construct four RL environments with misspecified rewards. We investigate reward hacking as a function of agent capabilities: model capacity, action space resolution, observation space noise, and training time. More capable agents often exploit reward misspecifications, achieving higher proxy reward and lower true reward than less capable agents. Moreover, we find instances of phase transitions: capability th"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2201.03544","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2201.03544/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-21T07:03:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7hL1Lfqz8RlM+zgL/FDIh3LQMy//6UHbIePMM+E5tatnRM+ioWuv6AdHu7Iis8avhguXKjTLHnvoRJyhuqNkAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-21T11:45:23.302307Z"},"content_sha256":"955ea131fa4160df68e8e2ebec844e0cf6f3a4918bb4247ef0539f879b1cb7c9","schema_version":"1.0","event_id":"sha256:955ea131fa4160df68e8e2ebec844e0cf6f3a4918bb4247ef0539f879b1cb7c9"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/PQQC7F4H7VGFOO4ZGUIJFAOAYL/bundle.json","state_url":"https://pith.science/pith/PQQC7F4H7VGFOO4ZGUIJFAOAYL/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/PQQC7F4H7VGFOO4ZGUIJFAOAYL/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-05-21T11:45:23Z","links":{"resolver":"https://pith.science/pith/PQQC7F4H7VGFOO4ZGUIJFAOAYL","bundle":"https://pith.science/pith/PQQC7F4H7VGFOO4ZGUIJFAOAYL/bundle.json","state":"https://pith.science/pith/PQQC7F4H7VGFOO4ZGUIJFAOAYL/state.json","well_known_bundle":"https://pith.science/.well-known/pith/PQQC7F4H7VGFOO4ZGUIJFAOAYL/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:PQQC7F4H7VGFOO4ZGUIJFAOAYL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"6913c36b9ab8b86c32aae9efedbf89d259fbf5449156aa83678a4066709e0c76","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2022-01-10T18:58:52Z","title_canon_sha256":"095490091c403c86f8c23378791367d7633e83c21fbfa36d10be2655d5908b8e"},"schema_version":"1.0","source":{"id":"2201.03544","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2201.03544","created_at":"2026-05-21T07:03:14Z"},{"alias_kind":"arxiv_version","alias_value":"2201.03544v2","created_at":"2026-05-21T07:03:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2201.03544","created_at":"2026-05-21T07:03:14Z"},{"alias_kind":"pith_short_12","alias_value":"PQQC7F4H7VGF","created_at":"2026-05-21T07:03:14Z"},{"alias_kind":"pith_short_16","alias_value":"PQQC7F4H7VGFOO4Z","created_at":"2026-05-21T07:03:14Z"},{"alias_kind":"pith_short_8","alias_value":"PQQC7F4H","created_at":"2026-05-21T07:03:14Z"}],"graph_snapshots":[{"event_id":"sha256:955ea131fa4160df68e8e2ebec844e0cf6f3a4918bb4247ef0539f879b1cb7c9","target":"graph","created_at":"2026-05-21T07:03:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2201.03544/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reward hacking -- where RL agents exploit gaps in misspecified reward functions -- has been widely observed, but not yet systematically studied. To understand how reward hacking arises, we construct four RL environments with misspecified rewards. We investigate reward hacking as a function of agent capabilities: model capacity, action space resolution, observation space noise, and training time. More capable agents often exploit reward misspecifications, achieving higher proxy reward and lower true reward than less capable agents. Moreover, we find instances of phase transitions: capability th","authors_text":"Alexander Pan, Jacob Steinhardt, Kush Bhatia","cross_cats":["cs.AI","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2022-01-10T18:58:52Z","title":"The Effects of Reward Misspecification: Mapping and Mitigating Misaligned Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2201.03544","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:78f6566c4aab50372360e924dc1aba304499cb079752da32ac9e35c7df349e77","target":"record","created_at":"2026-05-21T07:03:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"6913c36b9ab8b86c32aae9efedbf89d259fbf5449156aa83678a4066709e0c76","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2022-01-10T18:58:52Z","title_canon_sha256":"095490091c403c86f8c23378791367d7633e83c21fbfa36d10be2655d5908b8e"},"schema_version":"1.0","source":{"id":"2201.03544","kind":"arxiv","version":2}},"canonical_sha256":"7c202f9787fd4c573b9935109281c0c2f54a705f54b072d7ce644bb3c15a73c0","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7c202f9787fd4c573b9935109281c0c2f54a705f54b072d7ce644bb3c15a73c0","first_computed_at":"2026-05-21T07:03:14.352943Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-21T07:03:14.352943Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"gtbrrXowFli5ge/xaNPRtC4XDe8rr3G3FxBR3hbMFKEQ5Zvi380boKnbIfp+tTqMBlvk9qonWPm+Ar0nTRHXBw==","signature_status":"signed_v1","signed_at":"2026-05-21T07:03:14.354677Z","signed_message":"canonical_sha256_bytes"},"source_id":"2201.03544","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:78f6566c4aab50372360e924dc1aba304499cb079752da32ac9e35c7df349e77","sha256:955ea131fa4160df68e8e2ebec844e0cf6f3a4918bb4247ef0539f879b1cb7c9"],"state_sha256":"a54933ac3e5122b8e815501fa292cd9effa45d4d0c25be9cbd85e1cbf004d347"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"LxSNn63S6HvTG8O7Vt0Trki5ojJKxoS/Sx6jTSAb017w40+wYwmFOenG0zRifbWNP2h+hsVgg6mUbiYcjzvtBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-05-21T11:45:23.304862Z","bundle_sha256":"ffa97954319a83d3e506cd5421f0362548cb2c824f77858130a70a18f182a7d7"}}