{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:EZVLPNDGTSIHJJ7XVCJC7DBYGU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"51d663df4581af78daa6d7b5da7a24549ce70784d2f84742a9a8c1cd5ce5f1bb","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-03-14T01:51:35Z","title_canon_sha256":"7d17dbce99de0574beaeb93e7d78dd22eee3a29495a9875644e3748829f19509"},"schema_version":"1.0","source":{"id":"2403.09032","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2403.09032","created_at":"2026-07-05T09:54:25Z"},{"alias_kind":"arxiv_version","alias_value":"2403.09032v3","created_at":"2026-07-05T09:54:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.09032","created_at":"2026-07-05T09:54:25Z"},{"alias_kind":"pith_short_12","alias_value":"EZVLPNDGTSIH","created_at":"2026-07-05T09:54:25Z"},{"alias_kind":"pith_short_16","alias_value":"EZVLPNDGTSIHJJ7X","created_at":"2026-07-05T09:54:25Z"},{"alias_kind":"pith_short_8","alias_value":"EZVLPNDG","created_at":"2026-07-05T09:54:25Z"}],"graph_snapshots":[{"event_id":"sha256:983f91d5d4271123b23cd44562af4070205c7b3397a8fba81ed9624e30531feb","target":"graph","created_at":"2026-07-05T09:54:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2403.09032/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Evaluating the alignment of large language models (LLMs) with user-defined coding preferences is a challenging endeavour that requires a deep assessment of LLMs' outputs. Existing methods and benchmarks rely primarily on automated metrics and static analysis tools, which often fail to capture the nuances of user instructions and LLM outputs. To address this gap, we propose using the LLM-as-a-Judge methodology to evaluate the alignment of LLMs with coding preferences. Based on this approach, we present CodeUltraFeedback, a comprehensive dataset designed to facilitate the evaluation and improvem","authors_text":"Aton Kamanda, Houari Sahraoui, Martin Weyssow, Xin Zhou","cross_cats":["cs.CL","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-03-14T01:51:35Z","title":"CodeUltraFeedback: An LLM-as-a-Judge Dataset for Aligning Large Language Models to Coding Preferences"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.09032","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:0e028423127e41f3a3aacdb701c521482326e14e73cdea48127545678d158a88","target":"record","created_at":"2026-07-05T09:54:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"51d663df4581af78daa6d7b5da7a24549ce70784d2f84742a9a8c1cd5ce5f1bb","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-03-14T01:51:35Z","title_canon_sha256":"7d17dbce99de0574beaeb93e7d78dd22eee3a29495a9875644e3748829f19509"},"schema_version":"1.0","source":{"id":"2403.09032","kind":"arxiv","version":3}},"canonical_sha256":"266ab7b4669c9074a7f7a8922f8c38352ca5afa6bea797ef6ce8591e384b7354","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"266ab7b4669c9074a7f7a8922f8c38352ca5afa6bea797ef6ce8591e384b7354","first_computed_at":"2026-07-05T09:54:25.343963Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:54:25.343963Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"x1DCPxD8fubxvPHmc7rPz70YBAL5YI9HFhGsARzxdqzQb1hpIUnLgejefzdrjPcfw4YoBTkVHxwx6oTda1hqDA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:54:25.344506Z","signed_message":"canonical_sha256_bytes"},"source_id":"2403.09032","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:0e028423127e41f3a3aacdb701c521482326e14e73cdea48127545678d158a88","sha256:983f91d5d4271123b23cd44562af4070205c7b3397a8fba81ed9624e30531feb"],"state_sha256":"abce6b9fa5cf998bb31c7a419f65cc0ae8eee61d128443c297f4b622cc8cc21e"}