{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:NY3K4MEJTF32LY7CBDXGYYXCCM","short_pith_number":"pith:NY3K4MEJ","canonical_record":{"source":{"id":"2305.07248","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-05-12T04:47:02Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"a1692c1dc893b41affdc90e065a072ec99e79119175af9c070700dfe92733aaa","abstract_canon_sha256":"592540ea93c53abb5cc3dd3c3e85029907c664e067b96a67e39b7ed615c96eab"},"schema_version":"1.0"},"canonical_sha256":"6e36ae30899977a5e3e208ee6c62e2130daeb0ba27be9a12f08bc6ed173ff3b1","source":{"kind":"arxiv","id":"2305.07248","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.07248","created_at":"2026-07-05T06:09:24Z"},{"alias_kind":"arxiv_version","alias_value":"2305.07248v1","created_at":"2026-07-05T06:09:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.07248","created_at":"2026-07-05T06:09:24Z"},{"alias_kind":"pith_short_12","alias_value":"NY3K4MEJTF32","created_at":"2026-07-05T06:09:24Z"},{"alias_kind":"pith_short_16","alias_value":"NY3K4MEJTF32LY7C","created_at":"2026-07-05T06:09:24Z"},{"alias_kind":"pith_short_8","alias_value":"NY3K4MEJ","created_at":"2026-07-05T06:09:24Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:NY3K4MEJTF32LY7CBDXGYYXCCM","target":"record","payload":{"canonical_record":{"source":{"id":"2305.07248","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-05-12T04:47:02Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"a1692c1dc893b41affdc90e065a072ec99e79119175af9c070700dfe92733aaa","abstract_canon_sha256":"592540ea93c53abb5cc3dd3c3e85029907c664e067b96a67e39b7ed615c96eab"},"schema_version":"1.0"},"canonical_sha256":"6e36ae30899977a5e3e208ee6c62e2130daeb0ba27be9a12f08bc6ed173ff3b1","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:09:24.449561Z","signature_b64":"p+ATvTLGz40o7ZOSzDWgsFwEXF8Tr3O3FHnJoobqVZRi1IcRXrjXFy0mFRooLq2PDR3/CQ/kQHjqIn8mOE8fCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6e36ae30899977a5e3e208ee6c62e2130daeb0ba27be9a12f08bc6ed173ff3b1","last_reissued_at":"2026-07-05T06:09:24.449207Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:09:24.449207Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2305.07248","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:09:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rfwQZUJBhWEt5Z/N39oZj2RRsTveNrtzzWlPCuepwZWoE63ClRYUPgRCP7ZHOj82G4xGCz5CqbwjE81cIGA7Bw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-12T13:49:49.538675Z"},"content_sha256":"bead3c0bddbaba0d71ca3e680fc36c1736d7beb10d201e738b372c00b9d94c71","schema_version":"1.0","event_id":"sha256:bead3c0bddbaba0d71ca3e680fc36c1736d7beb10d201e738b372c00b9d94c71"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:NY3K4MEJTF32LY7CBDXGYYXCCM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Quantile-Based Deep Reinforcement Learning using Two-Timescale Policy Gradient Algorithms","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Jiaqiao Hu, Jinyang Jiang, Yijie Peng","submitted_at":"2023-05-12T04:47:02Z","abstract_excerpt":"Classical reinforcement learning (RL) aims to optimize the expected cumulative reward. In this work, we consider the RL setting where the goal is to optimize the quantile of the cumulative reward. We parameterize the policy controlling actions by neural networks, and propose a novel policy gradient algorithm called Quantile-Based Policy Optimization (QPO) and its variant Quantile-Based Proximal Policy Optimization (QPPO) for solving deep RL problems with quantile objectives. QPO uses two coupled iterations running at different timescales for simultaneously updating quantiles and policy paramet"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.07248","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.07248/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:09:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"fROSIS3YJltzBtbkC281OxlWnc4BJ03u7mNTghyf+0S7sacHJs/JEpI6kWs5rnu4rEB2CSsIYrm4Cj/VuupNAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-12T13:49:49.539193Z"},"content_sha256":"eb294579cc50623251473bc1002b1b748142344d13564613c402d4add4471691","schema_version":"1.0","event_id":"sha256:eb294579cc50623251473bc1002b1b748142344d13564613c402d4add4471691"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/NY3K4MEJTF32LY7CBDXGYYXCCM/bundle.json","state_url":"https://pith.science/pith/NY3K4MEJTF32LY7CBDXGYYXCCM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/NY3K4MEJTF32LY7CBDXGYYXCCM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-12T13:49:49Z","links":{"resolver":"https://pith.science/pith/NY3K4MEJTF32LY7CBDXGYYXCCM","bundle":"https://pith.science/pith/NY3K4MEJTF32LY7CBDXGYYXCCM/bundle.json","state":"https://pith.science/pith/NY3K4MEJTF32LY7CBDXGYYXCCM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/NY3K4MEJTF32LY7CBDXGYYXCCM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:NY3K4MEJTF32LY7CBDXGYYXCCM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"592540ea93c53abb5cc3dd3c3e85029907c664e067b96a67e39b7ed615c96eab","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-05-12T04:47:02Z","title_canon_sha256":"a1692c1dc893b41affdc90e065a072ec99e79119175af9c070700dfe92733aaa"},"schema_version":"1.0","source":{"id":"2305.07248","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.07248","created_at":"2026-07-05T06:09:24Z"},{"alias_kind":"arxiv_version","alias_value":"2305.07248v1","created_at":"2026-07-05T06:09:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.07248","created_at":"2026-07-05T06:09:24Z"},{"alias_kind":"pith_short_12","alias_value":"NY3K4MEJTF32","created_at":"2026-07-05T06:09:24Z"},{"alias_kind":"pith_short_16","alias_value":"NY3K4MEJTF32LY7C","created_at":"2026-07-05T06:09:24Z"},{"alias_kind":"pith_short_8","alias_value":"NY3K4MEJ","created_at":"2026-07-05T06:09:24Z"}],"graph_snapshots":[{"event_id":"sha256:eb294579cc50623251473bc1002b1b748142344d13564613c402d4add4471691","target":"graph","created_at":"2026-07-05T06:09:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2305.07248/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Classical reinforcement learning (RL) aims to optimize the expected cumulative reward. In this work, we consider the RL setting where the goal is to optimize the quantile of the cumulative reward. We parameterize the policy controlling actions by neural networks, and propose a novel policy gradient algorithm called Quantile-Based Policy Optimization (QPO) and its variant Quantile-Based Proximal Policy Optimization (QPPO) for solving deep RL problems with quantile objectives. QPO uses two coupled iterations running at different timescales for simultaneously updating quantiles and policy paramet","authors_text":"Jiaqiao Hu, Jinyang Jiang, Yijie Peng","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-05-12T04:47:02Z","title":"Quantile-Based Deep Reinforcement Learning using Two-Timescale Policy Gradient Algorithms"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.07248","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:bead3c0bddbaba0d71ca3e680fc36c1736d7beb10d201e738b372c00b9d94c71","target":"record","created_at":"2026-07-05T06:09:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"592540ea93c53abb5cc3dd3c3e85029907c664e067b96a67e39b7ed615c96eab","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-05-12T04:47:02Z","title_canon_sha256":"a1692c1dc893b41affdc90e065a072ec99e79119175af9c070700dfe92733aaa"},"schema_version":"1.0","source":{"id":"2305.07248","kind":"arxiv","version":1}},"canonical_sha256":"6e36ae30899977a5e3e208ee6c62e2130daeb0ba27be9a12f08bc6ed173ff3b1","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6e36ae30899977a5e3e208ee6c62e2130daeb0ba27be9a12f08bc6ed173ff3b1","first_computed_at":"2026-07-05T06:09:24.449207Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:09:24.449207Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"p+ATvTLGz40o7ZOSzDWgsFwEXF8Tr3O3FHnJoobqVZRi1IcRXrjXFy0mFRooLq2PDR3/CQ/kQHjqIn8mOE8fCw==","signature_status":"signed_v1","signed_at":"2026-07-05T06:09:24.449561Z","signed_message":"canonical_sha256_bytes"},"source_id":"2305.07248","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:bead3c0bddbaba0d71ca3e680fc36c1736d7beb10d201e738b372c00b9d94c71","sha256:eb294579cc50623251473bc1002b1b748142344d13564613c402d4add4471691"],"state_sha256":"9751637c45aecd886010cedc7c0eb8338e294f036d97f8b199af6761689a9257"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ZPNXrW/hNI4+lSmVReYdWilEsvmewLov1KkmvGPRU7QSU1wVw0MKUBirKdr4As60toWaXdDIpkj1bCQg1C7LDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-12T13:49:49.543299Z","bundle_sha256":"eee58483b509f06df801dededc7e92f1726f81199f974fb0a37f50a672d63a92"}}