{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:DQZWRNYPCJ5QDMQ7JFCT26DBZP","short_pith_number":"pith:DQZWRNYP","schema_version":"1.0","canonical_sha256":"1c3368b70f127b01b21f49453d7861cbe88a3bddfbb340ecc3806e2a9280ce42","source":{"kind":"arxiv","id":"2506.11902","version":1},"attestation_state":"computed","paper":{"title":"TreeRL: LLM Reinforcement Learning with On-Policy Tree Search","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Jie Tang, Rui Lu, Yujiang Li, Yuxiao Dong, Zhenyu Hou, Ziniu Hu","submitted_at":"2025-06-13T15:52:37Z","abstract_excerpt":"Reinforcement learning (RL) with tree search has demonstrated superior performance in traditional reasoning tasks. Compared to conventional independent chain sampling strategies with outcome supervision, tree search enables better exploration of the reasoning space and provides dense, on-policy process rewards during RL training but remains under-explored in On-Policy LLM RL. We propose TreeRL, a reinforcement learning framework that directly incorporates on-policy tree search for RL training. Our approach includes intermediate supervision and eliminates the need for a separate reward model tr"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2506.11902","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-13T15:52:37Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"2a747faa4fc974c86a1a7841837e6959c70e2f1cd2eef4cfe9dcf9e35a9d85f5","abstract_canon_sha256":"73e7f1fbc75196d42008fbe7f7f4d917ff6fe75ca82408a7b7fb58000ca8284f"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:21:12.584784Z","signature_b64":"VUvL8WuWzgBvAuCG8svcONU+h8ImPFp6WvNMpPrSG7HOV7modk9Qi7jzDjQcs9dC8jLPtO9rr9jMwDRvEFDvDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1c3368b70f127b01b21f49453d7861cbe88a3bddfbb340ecc3806e2a9280ce42","last_reissued_at":"2026-07-05T11:21:12.584308Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:21:12.584308Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"TreeRL: LLM Reinforcement Learning with On-Policy Tree Search","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Jie Tang, Rui Lu, Yujiang Li, Yuxiao Dong, Zhenyu Hou, Ziniu Hu","submitted_at":"2025-06-13T15:52:37Z","abstract_excerpt":"Reinforcement learning (RL) with tree search has demonstrated superior performance in traditional reasoning tasks. Compared to conventional independent chain sampling strategies with outcome supervision, tree search enables better exploration of the reasoning space and provides dense, on-policy process rewards during RL training but remains under-explored in On-Policy LLM RL. We propose TreeRL, a reinforcement learning framework that directly incorporates on-policy tree search for RL training. Our approach includes intermediate supervision and eliminates the need for a separate reward model tr"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.11902","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.11902/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2506.11902","created_at":"2026-07-05T11:21:12.584381+00:00"},{"alias_kind":"arxiv_version","alias_value":"2506.11902v1","created_at":"2026-07-05T11:21:12.584381+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.11902","created_at":"2026-07-05T11:21:12.584381+00:00"},{"alias_kind":"pith_short_12","alias_value":"DQZWRNYPCJ5Q","created_at":"2026-07-05T11:21:12.584381+00:00"},{"alias_kind":"pith_short_16","alias_value":"DQZWRNYPCJ5QDMQ7","created_at":"2026-07-05T11:21:12.584381+00:00"},{"alias_kind":"pith_short_8","alias_value":"DQZWRNYP","created_at":"2026-07-05T11:21:12.584381+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":13,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.05378","citing_title":"CompactionRL: Reinforcement Learning with Context Compaction for Long-Horizon Agents","ref_index":6,"is_internal_anchor":true},{"citing_arxiv_id":"2606.26453","citing_title":"Optimizing CUDA like a Human: Micro-Profiling Tools as Expert Surrogates for LLM-Based GPU Kernel Optimization","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2606.21943","citing_title":"Modularized Reinforcement Learning on LLMs: From MDP Creation to Exploration and Learning","ref_index":74,"is_internal_anchor":false},{"citing_arxiv_id":"2606.18089","citing_title":"From Reasoning Traces to Reusable Modules: Understanding Compositional Generalization in Language Model Reasoning","ref_index":128,"is_internal_anchor":false},{"citing_arxiv_id":"2606.03489","citing_title":"Learn from Your Mistakes: Tree-like Self-Play for Secure Code LLMs","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2606.01249","citing_title":"Trust Region On-Policy Distillation","ref_index":130,"is_internal_anchor":false},{"citing_arxiv_id":"2605.29697","citing_title":"Beyond Trajectory Rewards: Step-level Credit Assignment for Agentic Search via Graph Modeling","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2511.00413","citing_title":"Tree Training: Accelerating Agentic LLMs Training via Shared Prefix Reuse","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2509.08827","citing_title":"A Survey of Reinforcement Learning for Large Reasoning Models","ref_index":198,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08315","citing_title":"Reflective Prompted Policy Optimization: Trajectory-Grounded Revision and Salience Bias","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06200","citing_title":"A$^2$TGPO: Agentic Turn-Group Policy Optimization with Adaptive Turn-level Clipping","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2604.10827","citing_title":"Your Model Diversity, Not Method, Determines Reasoning Strategy","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2604.14518","citing_title":"Mind DeepResearch Technical Report","ref_index":12,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/DQZWRNYPCJ5QDMQ7JFCT26DBZP","json":"https://pith.science/pith/DQZWRNYPCJ5QDMQ7JFCT26DBZP.json","graph_json":"https://pith.science/api/pith-number/DQZWRNYPCJ5QDMQ7JFCT26DBZP/graph.json","events_json":"https://pith.science/api/pith-number/DQZWRNYPCJ5QDMQ7JFCT26DBZP/events.json","paper":"https://pith.science/paper/DQZWRNYP"},"agent_actions":{"view_html":"https://pith.science/pith/DQZWRNYPCJ5QDMQ7JFCT26DBZP","download_json":"https://pith.science/pith/DQZWRNYPCJ5QDMQ7JFCT26DBZP.json","view_paper":"https://pith.science/paper/DQZWRNYP","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2506.11902&json=true","fetch_graph":"https://pith.science/api/pith-number/DQZWRNYPCJ5QDMQ7JFCT26DBZP/graph.json","fetch_events":"https://pith.science/api/pith-number/DQZWRNYPCJ5QDMQ7JFCT26DBZP/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/DQZWRNYPCJ5QDMQ7JFCT26DBZP/action/timestamp_anchor","attest_storage":"https://pith.science/pith/DQZWRNYPCJ5QDMQ7JFCT26DBZP/action/storage_attestation","attest_author":"https://pith.science/pith/DQZWRNYPCJ5QDMQ7JFCT26DBZP/action/author_attestation","sign_citation":"https://pith.science/pith/DQZWRNYPCJ5QDMQ7JFCT26DBZP/action/citation_signature","submit_replication":"https://pith.science/pith/DQZWRNYPCJ5QDMQ7JFCT26DBZP/action/replication_record"}},"created_at":"2026-07-05T11:21:12.584381+00:00","updated_at":"2026-07-05T11:21:12.584381+00:00"}