{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2018:S2RCNKSA27TX3NKNRYQZIDXPSH","short_pith_number":"pith:S2RCNKSA","canonical_record":{"source":{"id":"1811.11214","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-11-27T19:32:27Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"ff1f4aab2dfc8332ea5f410aef927ebc3b0a67b5777ef354ebee45c071255654","abstract_canon_sha256":"34901392f774b10a527c38e01d0dcb7a1475ad44a44036c156bf84bddc1d48b7"},"schema_version":"1.0"},"canonical_sha256":"96a226aa40d7e77db54d8e21940eef91d03247b09e3e95cb09159dc9c95ee63a","source":{"kind":"arxiv","id":"1811.11214","version":5},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1811.11214","created_at":"2026-05-17T23:43:50Z"},{"alias_kind":"arxiv_version","alias_value":"1811.11214v5","created_at":"2026-05-17T23:43:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1811.11214","created_at":"2026-05-17T23:43:50Z"},{"alias_kind":"pith_short_12","alias_value":"S2RCNKSA27TX","created_at":"2026-05-18T12:32:50Z"},{"alias_kind":"pith_short_16","alias_value":"S2RCNKSA27TX3NKN","created_at":"2026-05-18T12:32:50Z"},{"alias_kind":"pith_short_8","alias_value":"S2RCNKSA","created_at":"2026-05-18T12:32:50Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2018:S2RCNKSA27TX3NKNRYQZIDXPSH","target":"record","payload":{"canonical_record":{"source":{"id":"1811.11214","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-11-27T19:32:27Z","cross_cats_sorted":["stat.ML"],"title_canon_sha256":"ff1f4aab2dfc8332ea5f410aef927ebc3b0a67b5777ef354ebee45c071255654","abstract_canon_sha256":"34901392f774b10a527c38e01d0dcb7a1475ad44a44036c156bf84bddc1d48b7"},"schema_version":"1.0"},"canonical_sha256":"96a226aa40d7e77db54d8e21940eef91d03247b09e3e95cb09159dc9c95ee63a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:43:50.444229Z","signature_b64":"2gqd/+a2lNoYFRTBq23OsTT4XWIbJzAV+4wdLNmGqn4rpoZLH4V0V5gXkQW+hzgUQuDOppkdeSa3kgDKTsDTAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"96a226aa40d7e77db54d8e21940eef91d03247b09e3e95cb09159dc9c95ee63a","last_reissued_at":"2026-05-17T23:43:50.443542Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:43:50.443542Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1811.11214","source_version":5,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:43:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"tPR5GohA5qAj594LuBIZun1lxlyfO01POFlicdiGPddjIvBSboheGD3vDAcCPcrfmX225e0Zr8sr32rv2xEnCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-02T23:02:03.458412Z"},"content_sha256":"86d67bd9efc3a18460514cf1c8256054a5bd44e90bf1351a2c4aecf62808d164","schema_version":"1.0","event_id":"sha256:86d67bd9efc3a18460514cf1c8256054a5bd44e90bf1351a2c4aecf62808d164"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2018:S2RCNKSA27TX3NKNRYQZIDXPSH","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Understanding the impact of entropy on policy optimization","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["stat.ML"],"primary_cat":"cs.LG","authors_text":"Dale Schuurmans, Mohammad Norouzi, Nicolas Le Roux, Zafarali Ahmed","submitted_at":"2018-11-27T19:32:27Z","abstract_excerpt":"Entropy regularization is commonly used to improve policy optimization in reinforcement learning. It is believed to help with \\emph{exploration} by encouraging the selection of more stochastic policies. In this work, we analyze this claim using new visualizations of the optimization landscape based on randomly perturbing the loss function. We first show that even with access to the exact gradient, policy optimization is difficult due to the geometry of the objective function. Then, we qualitatively show that in some environments, a policy with higher entropy can make the optimization landscape"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1811.11214","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:43:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"TUtovZQCrba3dSbSMFh763Y0f8asKuL8x7GV+5H16waMz8MiTdSF46T52HidQ68xeLv82pdVyJd4BmMXGPGeCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-02T23:02:03.458784Z"},"content_sha256":"23839952a6035e6e89e719a9b6258ac1afce1d2190d617756191c6e1b2fe0a5b","schema_version":"1.0","event_id":"sha256:23839952a6035e6e89e719a9b6258ac1afce1d2190d617756191c6e1b2fe0a5b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/S2RCNKSA27TX3NKNRYQZIDXPSH/bundle.json","state_url":"https://pith.science/pith/S2RCNKSA27TX3NKNRYQZIDXPSH/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/S2RCNKSA27TX3NKNRYQZIDXPSH/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-02T23:02:03Z","links":{"resolver":"https://pith.science/pith/S2RCNKSA27TX3NKNRYQZIDXPSH","bundle":"https://pith.science/pith/S2RCNKSA27TX3NKNRYQZIDXPSH/bundle.json","state":"https://pith.science/pith/S2RCNKSA27TX3NKNRYQZIDXPSH/state.json","well_known_bundle":"https://pith.science/.well-known/pith/S2RCNKSA27TX3NKNRYQZIDXPSH/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2018:S2RCNKSA27TX3NKNRYQZIDXPSH","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"34901392f774b10a527c38e01d0dcb7a1475ad44a44036c156bf84bddc1d48b7","cross_cats_sorted":["stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-11-27T19:32:27Z","title_canon_sha256":"ff1f4aab2dfc8332ea5f410aef927ebc3b0a67b5777ef354ebee45c071255654"},"schema_version":"1.0","source":{"id":"1811.11214","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1811.11214","created_at":"2026-05-17T23:43:50Z"},{"alias_kind":"arxiv_version","alias_value":"1811.11214v5","created_at":"2026-05-17T23:43:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1811.11214","created_at":"2026-05-17T23:43:50Z"},{"alias_kind":"pith_short_12","alias_value":"S2RCNKSA27TX","created_at":"2026-05-18T12:32:50Z"},{"alias_kind":"pith_short_16","alias_value":"S2RCNKSA27TX3NKN","created_at":"2026-05-18T12:32:50Z"},{"alias_kind":"pith_short_8","alias_value":"S2RCNKSA","created_at":"2026-05-18T12:32:50Z"}],"graph_snapshots":[{"event_id":"sha256:23839952a6035e6e89e719a9b6258ac1afce1d2190d617756191c6e1b2fe0a5b","target":"graph","created_at":"2026-05-17T23:43:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Entropy regularization is commonly used to improve policy optimization in reinforcement learning. It is believed to help with \\emph{exploration} by encouraging the selection of more stochastic policies. In this work, we analyze this claim using new visualizations of the optimization landscape based on randomly perturbing the loss function. We first show that even with access to the exact gradient, policy optimization is difficult due to the geometry of the objective function. Then, we qualitatively show that in some environments, a policy with higher entropy can make the optimization landscape","authors_text":"Dale Schuurmans, Mohammad Norouzi, Nicolas Le Roux, Zafarali Ahmed","cross_cats":["stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-11-27T19:32:27Z","title":"Understanding the impact of entropy on policy optimization"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1811.11214","kind":"arxiv","version":5},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:86d67bd9efc3a18460514cf1c8256054a5bd44e90bf1351a2c4aecf62808d164","target":"record","created_at":"2026-05-17T23:43:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"34901392f774b10a527c38e01d0dcb7a1475ad44a44036c156bf84bddc1d48b7","cross_cats_sorted":["stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-11-27T19:32:27Z","title_canon_sha256":"ff1f4aab2dfc8332ea5f410aef927ebc3b0a67b5777ef354ebee45c071255654"},"schema_version":"1.0","source":{"id":"1811.11214","kind":"arxiv","version":5}},"canonical_sha256":"96a226aa40d7e77db54d8e21940eef91d03247b09e3e95cb09159dc9c95ee63a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"96a226aa40d7e77db54d8e21940eef91d03247b09e3e95cb09159dc9c95ee63a","first_computed_at":"2026-05-17T23:43:50.443542Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:43:50.443542Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"2gqd/+a2lNoYFRTBq23OsTT4XWIbJzAV+4wdLNmGqn4rpoZLH4V0V5gXkQW+hzgUQuDOppkdeSa3kgDKTsDTAA==","signature_status":"signed_v1","signed_at":"2026-05-17T23:43:50.444229Z","signed_message":"canonical_sha256_bytes"},"source_id":"1811.11214","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:86d67bd9efc3a18460514cf1c8256054a5bd44e90bf1351a2c4aecf62808d164","sha256:23839952a6035e6e89e719a9b6258ac1afce1d2190d617756191c6e1b2fe0a5b"],"state_sha256":"5eb8b84905cbddb0f230affbf859468185dd61e4560f2f36ab811840bd9f4d23"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PomxSpwQXvfI+RlgrHdKgM3M+EANcmFyfLKoFTfDszs4L1OUdFwlrEAA4OuxLkWaRFAjhbdawOTP2fI+Bsw1CA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-02T23:02:03.461069Z","bundle_sha256":"ea5a43f3e77de19a059f9b7569b02fbaa5ec43bebd7712ffb50240e874cbce12"}}