{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:IXR4SN42J7WTZODWXXK34ZC3JB","short_pith_number":"pith:IXR4SN42","canonical_record":{"source":{"id":"2410.08847","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-11T14:22:44Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"683b73ad1c53362948fd8f19a9e0c03e4ebc61bbc01c999f742e391d48866ed3","abstract_canon_sha256":"3d24f8f90aa9ac0e6d5844b0824dda53f49d7131c908352c5ac5ac412ce79c74"},"schema_version":"1.0"},"canonical_sha256":"45e3c9379a4fed3cb876bdd5be645b48447ae37dab72c9dfc5dcf50f916656cf","source":{"kind":"arxiv","id":"2410.08847","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.08847","created_at":"2026-07-05T10:54:37Z"},{"alias_kind":"arxiv_version","alias_value":"2410.08847v4","created_at":"2026-07-05T10:54:37Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.08847","created_at":"2026-07-05T10:54:37Z"},{"alias_kind":"pith_short_12","alias_value":"IXR4SN42J7WT","created_at":"2026-07-05T10:54:37Z"},{"alias_kind":"pith_short_16","alias_value":"IXR4SN42J7WTZODW","created_at":"2026-07-05T10:54:37Z"},{"alias_kind":"pith_short_8","alias_value":"IXR4SN42","created_at":"2026-07-05T10:54:37Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:IXR4SN42J7WTZODWXXK34ZC3JB","target":"record","payload":{"canonical_record":{"source":{"id":"2410.08847","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-11T14:22:44Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"683b73ad1c53362948fd8f19a9e0c03e4ebc61bbc01c999f742e391d48866ed3","abstract_canon_sha256":"3d24f8f90aa9ac0e6d5844b0824dda53f49d7131c908352c5ac5ac412ce79c74"},"schema_version":"1.0"},"canonical_sha256":"45e3c9379a4fed3cb876bdd5be645b48447ae37dab72c9dfc5dcf50f916656cf","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:54:37.612392Z","signature_b64":"3bkbRYAlORQblSovcy5hP0XTQHVIGLqHfmnkydOGKXF6ZR4eM8KF3aOBiTz64SDInJ7hOAI2owDen1pjDLOZDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"45e3c9379a4fed3cb876bdd5be645b48447ae37dab72c9dfc5dcf50f916656cf","last_reissued_at":"2026-07-05T10:54:37.611891Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:54:37.611891Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.08847","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:54:37Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"gVJE1r21V0E8O2B4djkJjfCEkd4V/1iFzX4S3bOg5dwhItPgI1PbDHezD0MM2tJrM/NSuPKsCrmnTK8HkXDKAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T02:49:24.727962Z"},"content_sha256":"5b9bfce473b4154d49c978b22097e2e9e0f13d2989614e8919086de65cf7540e","schema_version":"1.0","event_id":"sha256:5b9bfce473b4154d49c978b22097e2e9e0f13d2989614e8919086de65cf7540e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:IXR4SN42J7WTZODWXXK34ZC3JB","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Unintentional Unalignment: Likelihood Displacement in Direct Preference Optimization","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","stat.ML"],"primary_cat":"cs.LG","authors_text":"Adithya Bhaskar, Boris Hanin, Danqi Chen, Noam Razin, Sadhika Malladi, Sanjeev Arora","submitted_at":"2024-10-11T14:22:44Z","abstract_excerpt":"Direct Preference Optimization (DPO) and its variants are increasingly used for aligning language models with human preferences. Although these methods are designed to teach a model to generate preferred responses more frequently relative to dispreferred responses, prior work has observed that the likelihood of preferred responses often decreases during training. The current work sheds light on the causes and implications of this counter-intuitive phenomenon, which we term likelihood displacement. We demonstrate that likelihood displacement can be catastrophic, shifting probability mass from p"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.08847","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.08847/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:54:37Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4dz6QLDC4qXQQGkbqxF6wGAqqyBYq6p368LvGfGam+3WOd1AWOqh0iIXTKF3SFDQmpvTjhGSNDAIoIHwhU9yDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T02:49:24.728354Z"},"content_sha256":"bfe5911f46f0fb87468ac4aa061df0f3ab4897de12e0f7438b16d4f8cbe0b0fc","schema_version":"1.0","event_id":"sha256:bfe5911f46f0fb87468ac4aa061df0f3ab4897de12e0f7438b16d4f8cbe0b0fc"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/IXR4SN42J7WTZODWXXK34ZC3JB/bundle.json","state_url":"https://pith.science/pith/IXR4SN42J7WTZODWXXK34ZC3JB/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/IXR4SN42J7WTZODWXXK34ZC3JB/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-09T02:49:24Z","links":{"resolver":"https://pith.science/pith/IXR4SN42J7WTZODWXXK34ZC3JB","bundle":"https://pith.science/pith/IXR4SN42J7WTZODWXXK34ZC3JB/bundle.json","state":"https://pith.science/pith/IXR4SN42J7WTZODWXXK34ZC3JB/state.json","well_known_bundle":"https://pith.science/.well-known/pith/IXR4SN42J7WTZODWXXK34ZC3JB/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:IXR4SN42J7WTZODWXXK34ZC3JB","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3d24f8f90aa9ac0e6d5844b0824dda53f49d7131c908352c5ac5ac412ce79c74","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-11T14:22:44Z","title_canon_sha256":"683b73ad1c53362948fd8f19a9e0c03e4ebc61bbc01c999f742e391d48866ed3"},"schema_version":"1.0","source":{"id":"2410.08847","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.08847","created_at":"2026-07-05T10:54:37Z"},{"alias_kind":"arxiv_version","alias_value":"2410.08847v4","created_at":"2026-07-05T10:54:37Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.08847","created_at":"2026-07-05T10:54:37Z"},{"alias_kind":"pith_short_12","alias_value":"IXR4SN42J7WT","created_at":"2026-07-05T10:54:37Z"},{"alias_kind":"pith_short_16","alias_value":"IXR4SN42J7WTZODW","created_at":"2026-07-05T10:54:37Z"},{"alias_kind":"pith_short_8","alias_value":"IXR4SN42","created_at":"2026-07-05T10:54:37Z"}],"graph_snapshots":[{"event_id":"sha256:bfe5911f46f0fb87468ac4aa061df0f3ab4897de12e0f7438b16d4f8cbe0b0fc","target":"graph","created_at":"2026-07-05T10:54:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.08847/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Direct Preference Optimization (DPO) and its variants are increasingly used for aligning language models with human preferences. Although these methods are designed to teach a model to generate preferred responses more frequently relative to dispreferred responses, prior work has observed that the likelihood of preferred responses often decreases during training. The current work sheds light on the causes and implications of this counter-intuitive phenomenon, which we term likelihood displacement. We demonstrate that likelihood displacement can be catastrophic, shifting probability mass from p","authors_text":"Adithya Bhaskar, Boris Hanin, Danqi Chen, Noam Razin, Sadhika Malladi, Sanjeev Arora","cross_cats":["cs.AI","cs.CL","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-11T14:22:44Z","title":"Unintentional Unalignment: Likelihood Displacement in Direct Preference Optimization"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.08847","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5b9bfce473b4154d49c978b22097e2e9e0f13d2989614e8919086de65cf7540e","target":"record","created_at":"2026-07-05T10:54:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3d24f8f90aa9ac0e6d5844b0824dda53f49d7131c908352c5ac5ac412ce79c74","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-11T14:22:44Z","title_canon_sha256":"683b73ad1c53362948fd8f19a9e0c03e4ebc61bbc01c999f742e391d48866ed3"},"schema_version":"1.0","source":{"id":"2410.08847","kind":"arxiv","version":4}},"canonical_sha256":"45e3c9379a4fed3cb876bdd5be645b48447ae37dab72c9dfc5dcf50f916656cf","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"45e3c9379a4fed3cb876bdd5be645b48447ae37dab72c9dfc5dcf50f916656cf","first_computed_at":"2026-07-05T10:54:37.611891Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:54:37.611891Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"3bkbRYAlORQblSovcy5hP0XTQHVIGLqHfmnkydOGKXF6ZR4eM8KF3aOBiTz64SDInJ7hOAI2owDen1pjDLOZDw==","signature_status":"signed_v1","signed_at":"2026-07-05T10:54:37.612392Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.08847","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5b9bfce473b4154d49c978b22097e2e9e0f13d2989614e8919086de65cf7540e","sha256:bfe5911f46f0fb87468ac4aa061df0f3ab4897de12e0f7438b16d4f8cbe0b0fc"],"state_sha256":"125993ab0a668990ab7c7226538d48d2e999aa2c4dbba05c23ac8c97dd8cebe1"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PXFPVgZi+141wUT/dHVDNZdcTL7wsF1wJ1sh95DQkJpoP48dgAEK/rvDChQ8ZR4NGL9Q2rYegVnj+9OJuermDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-09T02:49:24.730369Z","bundle_sha256":"0353a9b33694a32a1a63f8c17cffb890b8cdc78a1bcebae3790a9468f9f30fbb"}}