{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:TMN64O2ACQ2SZVMX6OKME5UIFC","short_pith_number":"pith:TMN64O2A","canonical_record":{"source":{"id":"2406.19644","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-06-28T04:21:24Z","cross_cats_sorted":[],"title_canon_sha256":"356889a8f0bd6da4ab9e18e7ec58b38b94a915f837e2a0eb81df22ae1788c360","abstract_canon_sha256":"7270ecd7bc97d72eabe29007b53219f623da7c6b7e2cb34809b7090156f6a441"},"schema_version":"1.0"},"canonical_sha256":"9b1bee3b4014352cd597f394c2768828be05c045060de9144a0d2ecbfe09d5ef","source":{"kind":"arxiv","id":"2406.19644","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.19644","created_at":"2026-07-05T08:38:12Z"},{"alias_kind":"arxiv_version","alias_value":"2406.19644v2","created_at":"2026-07-05T08:38:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.19644","created_at":"2026-07-05T08:38:12Z"},{"alias_kind":"pith_short_12","alias_value":"TMN64O2ACQ2S","created_at":"2026-07-05T08:38:12Z"},{"alias_kind":"pith_short_16","alias_value":"TMN64O2ACQ2SZVMX","created_at":"2026-07-05T08:38:12Z"},{"alias_kind":"pith_short_8","alias_value":"TMN64O2A","created_at":"2026-07-05T08:38:12Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:TMN64O2ACQ2SZVMX6OKME5UIFC","target":"record","payload":{"canonical_record":{"source":{"id":"2406.19644","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-06-28T04:21:24Z","cross_cats_sorted":[],"title_canon_sha256":"356889a8f0bd6da4ab9e18e7ec58b38b94a915f837e2a0eb81df22ae1788c360","abstract_canon_sha256":"7270ecd7bc97d72eabe29007b53219f623da7c6b7e2cb34809b7090156f6a441"},"schema_version":"1.0"},"canonical_sha256":"9b1bee3b4014352cd597f394c2768828be05c045060de9144a0d2ecbfe09d5ef","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:38:12.780027Z","signature_b64":"Uxsu+U7NP/cT4xvN0cGa3+CSQ+9ScnS3eUkVgMUZ+bu5AC5vt/uaibsr4Bs86uAbo4fnAs0boQeZBvTvJV0bBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"9b1bee3b4014352cd597f394c2768828be05c045060de9144a0d2ecbfe09d5ef","last_reissued_at":"2026-07-05T08:38:12.779499Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:38:12.779499Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.19644","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:38:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"q0kbUi46euYYW/IhX77BfVc/nJMpqx9dwmrjP/5BmqKa7K9Vq+FZ62XDs650mib57Dweqi7biMj50IgakA5hDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-08T12:59:52.681073Z"},"content_sha256":"7fba4df2d5f725d8cdf6824dd1ae76f9bcb1840aceb693046dc2a802b95c0bfa","schema_version":"1.0","event_id":"sha256:7fba4df2d5f725d8cdf6824dd1ae76f9bcb1840aceb693046dc2a802b95c0bfa"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:TMN64O2ACQ2SZVMX6OKME5UIFC","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Beyond Human Preferences: Exploring Reinforcement Learning Trajectory Evaluation and Improvement through LLMs","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Jianxin Li, Qingyun Sun, Shiqi Gao, Tianchen Zhu, Zichao Shen","submitted_at":"2024-06-28T04:21:24Z","abstract_excerpt":"Reinforcement learning (RL) faces challenges in evaluating policy trajectories within intricate game tasks due to the difficulty in designing comprehensive and precise reward functions. This inherent difficulty curtails the broader application of RL within game environments characterized by diverse constraints. Preference-based reinforcement learning (PbRL) presents a pioneering framework that capitalizes on human preferences as pivotal reward signals, thereby circumventing the need for meticulous reward engineering. However, obtaining preference data from human experts is costly and inefficie"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.19644","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.19644/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:38:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"tPK2E74Wgl3prKRfMc1nYBGwQm6GBMXow2fEbg3on5fClhZBVnHkKLe+q96eOeEQXnUKCXE3IQBUeRe/5lWNAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-08T12:59:52.681481Z"},"content_sha256":"c62ad4bd994564a2bf0a67a93a66deade76270a5c4639c49da91f01a87384ee5","schema_version":"1.0","event_id":"sha256:c62ad4bd994564a2bf0a67a93a66deade76270a5c4639c49da91f01a87384ee5"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/TMN64O2ACQ2SZVMX6OKME5UIFC/bundle.json","state_url":"https://pith.science/pith/TMN64O2ACQ2SZVMX6OKME5UIFC/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/TMN64O2ACQ2SZVMX6OKME5UIFC/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-08T12:59:52Z","links":{"resolver":"https://pith.science/pith/TMN64O2ACQ2SZVMX6OKME5UIFC","bundle":"https://pith.science/pith/TMN64O2ACQ2SZVMX6OKME5UIFC/bundle.json","state":"https://pith.science/pith/TMN64O2ACQ2SZVMX6OKME5UIFC/state.json","well_known_bundle":"https://pith.science/.well-known/pith/TMN64O2ACQ2SZVMX6OKME5UIFC/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:TMN64O2ACQ2SZVMX6OKME5UIFC","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7270ecd7bc97d72eabe29007b53219f623da7c6b7e2cb34809b7090156f6a441","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-06-28T04:21:24Z","title_canon_sha256":"356889a8f0bd6da4ab9e18e7ec58b38b94a915f837e2a0eb81df22ae1788c360"},"schema_version":"1.0","source":{"id":"2406.19644","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.19644","created_at":"2026-07-05T08:38:12Z"},{"alias_kind":"arxiv_version","alias_value":"2406.19644v2","created_at":"2026-07-05T08:38:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.19644","created_at":"2026-07-05T08:38:12Z"},{"alias_kind":"pith_short_12","alias_value":"TMN64O2ACQ2S","created_at":"2026-07-05T08:38:12Z"},{"alias_kind":"pith_short_16","alias_value":"TMN64O2ACQ2SZVMX","created_at":"2026-07-05T08:38:12Z"},{"alias_kind":"pith_short_8","alias_value":"TMN64O2A","created_at":"2026-07-05T08:38:12Z"}],"graph_snapshots":[{"event_id":"sha256:c62ad4bd994564a2bf0a67a93a66deade76270a5c4639c49da91f01a87384ee5","target":"graph","created_at":"2026-07-05T08:38:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.19644/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement learning (RL) faces challenges in evaluating policy trajectories within intricate game tasks due to the difficulty in designing comprehensive and precise reward functions. This inherent difficulty curtails the broader application of RL within game environments characterized by diverse constraints. Preference-based reinforcement learning (PbRL) presents a pioneering framework that capitalizes on human preferences as pivotal reward signals, thereby circumventing the need for meticulous reward engineering. However, obtaining preference data from human experts is costly and inefficie","authors_text":"Jianxin Li, Qingyun Sun, Shiqi Gao, Tianchen Zhu, Zichao Shen","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-06-28T04:21:24Z","title":"Beyond Human Preferences: Exploring Reinforcement Learning Trajectory Evaluation and Improvement through LLMs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.19644","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:7fba4df2d5f725d8cdf6824dd1ae76f9bcb1840aceb693046dc2a802b95c0bfa","target":"record","created_at":"2026-07-05T08:38:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7270ecd7bc97d72eabe29007b53219f623da7c6b7e2cb34809b7090156f6a441","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-06-28T04:21:24Z","title_canon_sha256":"356889a8f0bd6da4ab9e18e7ec58b38b94a915f837e2a0eb81df22ae1788c360"},"schema_version":"1.0","source":{"id":"2406.19644","kind":"arxiv","version":2}},"canonical_sha256":"9b1bee3b4014352cd597f394c2768828be05c045060de9144a0d2ecbfe09d5ef","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"9b1bee3b4014352cd597f394c2768828be05c045060de9144a0d2ecbfe09d5ef","first_computed_at":"2026-07-05T08:38:12.779499Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:38:12.779499Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Uxsu+U7NP/cT4xvN0cGa3+CSQ+9ScnS3eUkVgMUZ+bu5AC5vt/uaibsr4Bs86uAbo4fnAs0boQeZBvTvJV0bBA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:38:12.780027Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.19644","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:7fba4df2d5f725d8cdf6824dd1ae76f9bcb1840aceb693046dc2a802b95c0bfa","sha256:c62ad4bd994564a2bf0a67a93a66deade76270a5c4639c49da91f01a87384ee5"],"state_sha256":"59eb33ab3a2ef7c3550e665a2127c75e55b8283870a069cadf3de9cdbdc04021"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kCbwTFnIE7aWrvunQEyy18+g/Q3y8kvqt8UI1ym6XwtmxBRJO/wdNDOPjJORMrvAzKHVBxWBssT1zAhQ29vNBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-08T12:59:52.684002Z","bundle_sha256":"f443fa833e9afcd3668e6c84af165f5c5c14128dcfe6b16402a28538e9685a53"}}