{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:JX3YDFAOKQTE7PDK2DACKLDHOZ","short_pith_number":"pith:JX3YDFAO","schema_version":"1.0","canonical_sha256":"4df781940e54264fbc6ad0c0252c677673bb4819fa3403393c73d0edadb23fcf","source":{"kind":"arxiv","id":"2501.06248","version":2},"attestation_state":"computed","paper":{"title":"Utility-inspired Reward Transformations Improve Reinforcement Learning Training of Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL","econ.GN","q-fin.EC"],"primary_cat":"cs.LG","authors_text":"Chirag Nagpal, Francesco Visin, Roberto-Rafael Maura-Rivero, Roma Patel","submitted_at":"2025-01-08T19:03:17Z","abstract_excerpt":"Current methods that train large language models (LLMs) with reinforcement learning feedback, often resort to averaging outputs of multiple rewards functions during training. This overlooks crucial aspects of individual reward dimensions and inter-reward dependencies that can lead to sub-optimal outcomes in generations. In this work, we show how linear aggregation of rewards exhibits some vulnerabilities that can lead to undesired properties of generated text. We then propose a transformation of reward functions inspired by economic theory of utility functions (specifically Inada conditions), "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2501.06248","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-01-08T19:03:17Z","cross_cats_sorted":["cs.AI","cs.CL","econ.GN","q-fin.EC"],"title_canon_sha256":"39904c52ea3d57ac26044b2d126c77a72380705f87a77115a602ffe5ec761668","abstract_canon_sha256":"c82c525a7be0b6478d9d71f608c78b75f3977aa2c77df28cd7c91ed9e54b7642"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:19:35.690641Z","signature_b64":"ZawAcGI2AFS7VTUCbEGHvZ/NGSUCmBnRPthihVls1S6z/kXFCC1U8ciXhKma/e6pSC4Ou/rz2TEYewMnBHH3Cg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4df781940e54264fbc6ad0c0252c677673bb4819fa3403393c73d0edadb23fcf","last_reissued_at":"2026-07-05T10:19:35.690206Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:19:35.690206Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Utility-inspired Reward Transformations Improve Reinforcement Learning Training of Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL","econ.GN","q-fin.EC"],"primary_cat":"cs.LG","authors_text":"Chirag Nagpal, Francesco Visin, Roberto-Rafael Maura-Rivero, Roma Patel","submitted_at":"2025-01-08T19:03:17Z","abstract_excerpt":"Current methods that train large language models (LLMs) with reinforcement learning feedback, often resort to averaging outputs of multiple rewards functions during training. This overlooks crucial aspects of individual reward dimensions and inter-reward dependencies that can lead to sub-optimal outcomes in generations. In this work, we show how linear aggregation of rewards exhibits some vulnerabilities that can lead to undesired properties of generated text. We then propose a transformation of reward functions inspired by economic theory of utility functions (specifically Inada conditions), "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.06248","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2501.06248/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2501.06248","created_at":"2026-07-05T10:19:35.690266+00:00"},{"alias_kind":"arxiv_version","alias_value":"2501.06248v2","created_at":"2026-07-05T10:19:35.690266+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.06248","created_at":"2026-07-05T10:19:35.690266+00:00"},{"alias_kind":"pith_short_12","alias_value":"JX3YDFAOKQTE","created_at":"2026-07-05T10:19:35.690266+00:00"},{"alias_kind":"pith_short_16","alias_value":"JX3YDFAOKQTE7PDK","created_at":"2026-07-05T10:19:35.690266+00:00"},{"alias_kind":"pith_short_8","alias_value":"JX3YDFAO","created_at":"2026-07-05T10:19:35.690266+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/JX3YDFAOKQTE7PDK2DACKLDHOZ","json":"https://pith.science/pith/JX3YDFAOKQTE7PDK2DACKLDHOZ.json","graph_json":"https://pith.science/api/pith-number/JX3YDFAOKQTE7PDK2DACKLDHOZ/graph.json","events_json":"https://pith.science/api/pith-number/JX3YDFAOKQTE7PDK2DACKLDHOZ/events.json","paper":"https://pith.science/paper/JX3YDFAO"},"agent_actions":{"view_html":"https://pith.science/pith/JX3YDFAOKQTE7PDK2DACKLDHOZ","download_json":"https://pith.science/pith/JX3YDFAOKQTE7PDK2DACKLDHOZ.json","view_paper":"https://pith.science/paper/JX3YDFAO","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2501.06248&json=true","fetch_graph":"https://pith.science/api/pith-number/JX3YDFAOKQTE7PDK2DACKLDHOZ/graph.json","fetch_events":"https://pith.science/api/pith-number/JX3YDFAOKQTE7PDK2DACKLDHOZ/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/JX3YDFAOKQTE7PDK2DACKLDHOZ/action/timestamp_anchor","attest_storage":"https://pith.science/pith/JX3YDFAOKQTE7PDK2DACKLDHOZ/action/storage_attestation","attest_author":"https://pith.science/pith/JX3YDFAOKQTE7PDK2DACKLDHOZ/action/author_attestation","sign_citation":"https://pith.science/pith/JX3YDFAOKQTE7PDK2DACKLDHOZ/action/citation_signature","submit_replication":"https://pith.science/pith/JX3YDFAOKQTE7PDK2DACKLDHOZ/action/replication_record"}},"created_at":"2026-07-05T10:19:35.690266+00:00","updated_at":"2026-07-05T10:19:35.690266+00:00"}