{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:IXR4SN42J7WTZODWXXK34ZC3JB","short_pith_number":"pith:IXR4SN42","schema_version":"1.0","canonical_sha256":"45e3c9379a4fed3cb876bdd5be645b48447ae37dab72c9dfc5dcf50f916656cf","source":{"kind":"arxiv","id":"2410.08847","version":4},"attestation_state":"computed","paper":{"title":"Unintentional Unalignment: Likelihood Displacement in Direct Preference Optimization","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","stat.ML"],"primary_cat":"cs.LG","authors_text":"Adithya Bhaskar, Boris Hanin, Danqi Chen, Noam Razin, Sadhika Malladi, Sanjeev Arora","submitted_at":"2024-10-11T14:22:44Z","abstract_excerpt":"Direct Preference Optimization (DPO) and its variants are increasingly used for aligning language models with human preferences. Although these methods are designed to teach a model to generate preferred responses more frequently relative to dispreferred responses, prior work has observed that the likelihood of preferred responses often decreases during training. The current work sheds light on the causes and implications of this counter-intuitive phenomenon, which we term likelihood displacement. We demonstrate that likelihood displacement can be catastrophic, shifting probability mass from p"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2410.08847","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-11T14:22:44Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"683b73ad1c53362948fd8f19a9e0c03e4ebc61bbc01c999f742e391d48866ed3","abstract_canon_sha256":"3d24f8f90aa9ac0e6d5844b0824dda53f49d7131c908352c5ac5ac412ce79c74"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:54:37.612392Z","signature_b64":"3bkbRYAlORQblSovcy5hP0XTQHVIGLqHfmnkydOGKXF6ZR4eM8KF3aOBiTz64SDInJ7hOAI2owDen1pjDLOZDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"45e3c9379a4fed3cb876bdd5be645b48447ae37dab72c9dfc5dcf50f916656cf","last_reissued_at":"2026-07-05T10:54:37.611891Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:54:37.611891Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Unintentional Unalignment: Likelihood Displacement in Direct Preference Optimization","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","stat.ML"],"primary_cat":"cs.LG","authors_text":"Adithya Bhaskar, Boris Hanin, Danqi Chen, Noam Razin, Sadhika Malladi, Sanjeev Arora","submitted_at":"2024-10-11T14:22:44Z","abstract_excerpt":"Direct Preference Optimization (DPO) and its variants are increasingly used for aligning language models with human preferences. Although these methods are designed to teach a model to generate preferred responses more frequently relative to dispreferred responses, prior work has observed that the likelihood of preferred responses often decreases during training. The current work sheds light on the causes and implications of this counter-intuitive phenomenon, which we term likelihood displacement. We demonstrate that likelihood displacement can be catastrophic, shifting probability mass from p"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.08847","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.08847/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.08847","created_at":"2026-07-05T10:54:37.611954+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.08847v4","created_at":"2026-07-05T10:54:37.611954+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.08847","created_at":"2026-07-05T10:54:37.611954+00:00"},{"alias_kind":"pith_short_12","alias_value":"IXR4SN42J7WT","created_at":"2026-07-05T10:54:37.611954+00:00"},{"alias_kind":"pith_short_16","alias_value":"IXR4SN42J7WTZODW","created_at":"2026-07-05T10:54:37.611954+00:00"},{"alias_kind":"pith_short_8","alias_value":"IXR4SN42","created_at":"2026-07-05T10:54:37.611954+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.06667","citing_title":"The Piggyback Hypothesis of Generalization: Explaining and Mitigating Emergent Misalignment","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2605.23961","citing_title":"Multimodal Alignment and Preference Optimization for Zero-Shot Conditional RNA Generation","ref_index":72,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08666","citing_title":"The Cancellation Hypothesis in Critic-Free RL: From Outcome Rewards to Token Credits","ref_index":15,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/IXR4SN42J7WTZODWXXK34ZC3JB","json":"https://pith.science/pith/IXR4SN42J7WTZODWXXK34ZC3JB.json","graph_json":"https://pith.science/api/pith-number/IXR4SN42J7WTZODWXXK34ZC3JB/graph.json","events_json":"https://pith.science/api/pith-number/IXR4SN42J7WTZODWXXK34ZC3JB/events.json","paper":"https://pith.science/paper/IXR4SN42"},"agent_actions":{"view_html":"https://pith.science/pith/IXR4SN42J7WTZODWXXK34ZC3JB","download_json":"https://pith.science/pith/IXR4SN42J7WTZODWXXK34ZC3JB.json","view_paper":"https://pith.science/paper/IXR4SN42","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.08847&json=true","fetch_graph":"https://pith.science/api/pith-number/IXR4SN42J7WTZODWXXK34ZC3JB/graph.json","fetch_events":"https://pith.science/api/pith-number/IXR4SN42J7WTZODWXXK34ZC3JB/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/IXR4SN42J7WTZODWXXK34ZC3JB/action/timestamp_anchor","attest_storage":"https://pith.science/pith/IXR4SN42J7WTZODWXXK34ZC3JB/action/storage_attestation","attest_author":"https://pith.science/pith/IXR4SN42J7WTZODWXXK34ZC3JB/action/author_attestation","sign_citation":"https://pith.science/pith/IXR4SN42J7WTZODWXXK34ZC3JB/action/citation_signature","submit_replication":"https://pith.science/pith/IXR4SN42J7WTZODWXXK34ZC3JB/action/replication_record"}},"created_at":"2026-07-05T10:54:37.611954+00:00","updated_at":"2026-07-05T10:54:37.611954+00:00"}