{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:AFEED2BOFPXAOB2BNNNQOIVHAS","short_pith_number":"pith:AFEED2BO","canonical_record":{"source":{"id":"2410.20176","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-26T13:12:27Z","cross_cats_sorted":[],"title_canon_sha256":"6331f113594ead5fe0fd11b168471189d6e024b27f78b8880a44e751aacedc33","abstract_canon_sha256":"232cf5955bc8a46dc0aa0fe6890e7e638f0ddd3c91b40983c1ca797b09607acd"},"schema_version":"1.0"},"canonical_sha256":"014841e82e2bee0707416b5b0722a70489e78ff704a1f5f09ed16149831f9a13","source":{"kind":"arxiv","id":"2410.20176","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.20176","created_at":"2026-07-05T09:26:40Z"},{"alias_kind":"arxiv_version","alias_value":"2410.20176v1","created_at":"2026-07-05T09:26:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.20176","created_at":"2026-07-05T09:26:40Z"},{"alias_kind":"pith_short_12","alias_value":"AFEED2BOFPXA","created_at":"2026-07-05T09:26:40Z"},{"alias_kind":"pith_short_16","alias_value":"AFEED2BOFPXAOB2B","created_at":"2026-07-05T09:26:40Z"},{"alias_kind":"pith_short_8","alias_value":"AFEED2BO","created_at":"2026-07-05T09:26:40Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:AFEED2BOFPXAOB2BNNNQOIVHAS","target":"record","payload":{"canonical_record":{"source":{"id":"2410.20176","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-26T13:12:27Z","cross_cats_sorted":[],"title_canon_sha256":"6331f113594ead5fe0fd11b168471189d6e024b27f78b8880a44e751aacedc33","abstract_canon_sha256":"232cf5955bc8a46dc0aa0fe6890e7e638f0ddd3c91b40983c1ca797b09607acd"},"schema_version":"1.0"},"canonical_sha256":"014841e82e2bee0707416b5b0722a70489e78ff704a1f5f09ed16149831f9a13","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:26:40.289553Z","signature_b64":"nH7boI84Bh0aMRse6XcTa4VCZ15/5WWSh+/G2rlT2zav1GH/Dn2NNCdMlQNXVrPPQAvJxtt+5o43YwCNJztGBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"014841e82e2bee0707416b5b0722a70489e78ff704a1f5f09ed16149831f9a13","last_reissued_at":"2026-07-05T09:26:40.289065Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:26:40.289065Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.20176","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:26:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ocxYlL09XoN1qnu3vzVKb2lBTkHkve5RrEZhjsHaawp9MYJ/7fTuRd5MvhWaaU4oNDBXmCDmJVarK1waWGbmDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T11:30:45.702409Z"},"content_sha256":"42006c52daa8c761b322dc3e69f00266008f8ad8b1424459a19a7e85745b5fb8","schema_version":"1.0","event_id":"sha256:42006c52daa8c761b322dc3e69f00266008f8ad8b1424459a19a7e85745b5fb8"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:AFEED2BOFPXAOB2BNNNQOIVHAS","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Beyond Simple Sum of Delayed Rewards: Non-Markovian Reward Modeling for Reinforcement Learning","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Jing-cheng Pang, Masashi Sugiyama, Qiyu Wu, Xin-Qiang Cai, Yao-Xiang Ding, Yuting Tang","submitted_at":"2024-10-26T13:12:27Z","abstract_excerpt":"Reinforcement Learning (RL) empowers agents to acquire various skills by learning from reward signals. Unfortunately, designing high-quality instance-level rewards often demands significant effort. An emerging alternative, RL with delayed reward, focuses on learning from rewards presented periodically, which can be obtained from human evaluators assessing the agent's performance over sequences of behaviors. However, traditional methods in this domain assume the existence of underlying Markovian rewards and that the observed delayed reward is simply the sum of instance-level rewards, both of wh"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.20176","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.20176/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:26:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"D4CKu76GKdSaPEPPmb5imPMiZj0jdTvVx6QklLrz4LLTl87nCanLyyjVZjc5aTKj4yL6gVPZTgp1Uw6ZfTCkCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T11:30:45.702781Z"},"content_sha256":"837eea5a5e8004b83b2ee8cfa15e26840266eb6abaafcd4bda44eceb63776057","schema_version":"1.0","event_id":"sha256:837eea5a5e8004b83b2ee8cfa15e26840266eb6abaafcd4bda44eceb63776057"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/AFEED2BOFPXAOB2BNNNQOIVHAS/bundle.json","state_url":"https://pith.science/pith/AFEED2BOFPXAOB2BNNNQOIVHAS/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/AFEED2BOFPXAOB2BNNNQOIVHAS/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T11:30:45Z","links":{"resolver":"https://pith.science/pith/AFEED2BOFPXAOB2BNNNQOIVHAS","bundle":"https://pith.science/pith/AFEED2BOFPXAOB2BNNNQOIVHAS/bundle.json","state":"https://pith.science/pith/AFEED2BOFPXAOB2BNNNQOIVHAS/state.json","well_known_bundle":"https://pith.science/.well-known/pith/AFEED2BOFPXAOB2BNNNQOIVHAS/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:AFEED2BOFPXAOB2BNNNQOIVHAS","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"232cf5955bc8a46dc0aa0fe6890e7e638f0ddd3c91b40983c1ca797b09607acd","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-26T13:12:27Z","title_canon_sha256":"6331f113594ead5fe0fd11b168471189d6e024b27f78b8880a44e751aacedc33"},"schema_version":"1.0","source":{"id":"2410.20176","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.20176","created_at":"2026-07-05T09:26:40Z"},{"alias_kind":"arxiv_version","alias_value":"2410.20176v1","created_at":"2026-07-05T09:26:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.20176","created_at":"2026-07-05T09:26:40Z"},{"alias_kind":"pith_short_12","alias_value":"AFEED2BOFPXA","created_at":"2026-07-05T09:26:40Z"},{"alias_kind":"pith_short_16","alias_value":"AFEED2BOFPXAOB2B","created_at":"2026-07-05T09:26:40Z"},{"alias_kind":"pith_short_8","alias_value":"AFEED2BO","created_at":"2026-07-05T09:26:40Z"}],"graph_snapshots":[{"event_id":"sha256:837eea5a5e8004b83b2ee8cfa15e26840266eb6abaafcd4bda44eceb63776057","target":"graph","created_at":"2026-07-05T09:26:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.20176/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement Learning (RL) empowers agents to acquire various skills by learning from reward signals. Unfortunately, designing high-quality instance-level rewards often demands significant effort. An emerging alternative, RL with delayed reward, focuses on learning from rewards presented periodically, which can be obtained from human evaluators assessing the agent's performance over sequences of behaviors. However, traditional methods in this domain assume the existence of underlying Markovian rewards and that the observed delayed reward is simply the sum of instance-level rewards, both of wh","authors_text":"Jing-cheng Pang, Masashi Sugiyama, Qiyu Wu, Xin-Qiang Cai, Yao-Xiang Ding, Yuting Tang","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-26T13:12:27Z","title":"Beyond Simple Sum of Delayed Rewards: Non-Markovian Reward Modeling for Reinforcement Learning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.20176","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:42006c52daa8c761b322dc3e69f00266008f8ad8b1424459a19a7e85745b5fb8","target":"record","created_at":"2026-07-05T09:26:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"232cf5955bc8a46dc0aa0fe6890e7e638f0ddd3c91b40983c1ca797b09607acd","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-26T13:12:27Z","title_canon_sha256":"6331f113594ead5fe0fd11b168471189d6e024b27f78b8880a44e751aacedc33"},"schema_version":"1.0","source":{"id":"2410.20176","kind":"arxiv","version":1}},"canonical_sha256":"014841e82e2bee0707416b5b0722a70489e78ff704a1f5f09ed16149831f9a13","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"014841e82e2bee0707416b5b0722a70489e78ff704a1f5f09ed16149831f9a13","first_computed_at":"2026-07-05T09:26:40.289065Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:26:40.289065Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"nH7boI84Bh0aMRse6XcTa4VCZ15/5WWSh+/G2rlT2zav1GH/Dn2NNCdMlQNXVrPPQAvJxtt+5o43YwCNJztGBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:26:40.289553Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.20176","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:42006c52daa8c761b322dc3e69f00266008f8ad8b1424459a19a7e85745b5fb8","sha256:837eea5a5e8004b83b2ee8cfa15e26840266eb6abaafcd4bda44eceb63776057"],"state_sha256":"e9943f07854d004f7ea8dbdbb623dd26a283ba6d733519e5dfc15d544a36fa44"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"EJuSkr63wLgXlSaiLljAzVd2ut1RanhJlIrN5SbSqmxpAux0zFNfpO8PSNJqp3uy08Jm/AoxIEN+cQdvR9vQCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T11:30:45.704785Z","bundle_sha256":"b84ee35d43155a166d637e6402c847de3af869a804296a737c3fcc3d20d8587e"}}