{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:RUHEZA2OOUJ3XGZHSWP5SBD5JW","short_pith_number":"pith:RUHEZA2O","canonical_record":{"source":{"id":"2410.07137","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-09T17:53:06Z","cross_cats_sorted":["cs.AI","cs.CR","cs.LG"],"title_canon_sha256":"eed0ed41615fb11ba8cb4a8fc8c1906fd302006012639b7ac71f828a6785c8ac","abstract_canon_sha256":"df3932b17b7a92e5e79a49548f24f2f492c4e4a49bed9f99d77648a4f9810152"},"schema_version":"1.0"},"canonical_sha256":"8d0e4c834e7513bb9b27959fd9047d4d9cdc2c5adbaf49736aa271b8900650de","source":{"kind":"arxiv","id":"2410.07137","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.07137","created_at":"2026-07-05T10:22:06Z"},{"alias_kind":"arxiv_version","alias_value":"2410.07137v2","created_at":"2026-07-05T10:22:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.07137","created_at":"2026-07-05T10:22:06Z"},{"alias_kind":"pith_short_12","alias_value":"RUHEZA2OOUJ3","created_at":"2026-07-05T10:22:06Z"},{"alias_kind":"pith_short_16","alias_value":"RUHEZA2OOUJ3XGZH","created_at":"2026-07-05T10:22:06Z"},{"alias_kind":"pith_short_8","alias_value":"RUHEZA2O","created_at":"2026-07-05T10:22:06Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:RUHEZA2OOUJ3XGZHSWP5SBD5JW","target":"record","payload":{"canonical_record":{"source":{"id":"2410.07137","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-09T17:53:06Z","cross_cats_sorted":["cs.AI","cs.CR","cs.LG"],"title_canon_sha256":"eed0ed41615fb11ba8cb4a8fc8c1906fd302006012639b7ac71f828a6785c8ac","abstract_canon_sha256":"df3932b17b7a92e5e79a49548f24f2f492c4e4a49bed9f99d77648a4f9810152"},"schema_version":"1.0"},"canonical_sha256":"8d0e4c834e7513bb9b27959fd9047d4d9cdc2c5adbaf49736aa271b8900650de","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:22:06.995710Z","signature_b64":"fx9I7gy3k0qQsacnDPoZZlpCAIoKIrmxrzq4VSPp0xGPiC+K89ooHcaHrjTGl8PApjTjhFq5bXZAxcrNgzw+Cg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8d0e4c834e7513bb9b27959fd9047d4d9cdc2c5adbaf49736aa271b8900650de","last_reissued_at":"2026-07-05T10:22:06.995158Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:22:06.995158Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.07137","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:22:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Acqg9iHpMVKWC3lY8F7v7Km+JxGfvd42Y3ZaLCojS5BTEzg0GFO8+y7f9KvhPncuiWTUVrOusivgqMFaE+cvCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-08T07:18:31.001566Z"},"content_sha256":"bb666fe9901dc8bf22f1fba4df005275b8c99b4a3ed16d196fc3dfd81402a3c7","schema_version":"1.0","event_id":"sha256:bb666fe9901dc8bf22f1fba4df005275b8c99b4a3ed16d196fc3dfd81402a3c7"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:RUHEZA2OOUJ3XGZHSWP5SBD5JW","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Cheating Automatic LLM Benchmarks: Null Models Achieve High Win Rates","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CR","cs.LG"],"primary_cat":"cs.CL","authors_text":"Chao Du, Jing Jiang, Min Lin, Qian Liu, Tianyu Pang, Xiaosen Zheng","submitted_at":"2024-10-09T17:53:06Z","abstract_excerpt":"Automatic LLM benchmarks, such as AlpacaEval 2.0, Arena-Hard-Auto, and MT-Bench, have become popular for evaluating language models due to their cost-effectiveness and scalability compared to human evaluation. Achieving high win rates on these benchmarks can significantly boost the promotional impact of newly released language models. This promotional benefit may motivate tricks, such as manipulating model output length or style to game win rates, even though several mechanisms have been developed to control length and disentangle style to reduce gameability. Nonetheless, we show that even a \""},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.07137","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.07137/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:22:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"qbfnPI2hPjWaXpUVMJVeacMij9C6+AhxDLnBJlGEC0pIhMF0PupVFGV4Ap3jMGVuHYRRn9XXmm229j1UhlRCCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-08T07:18:31.002202Z"},"content_sha256":"de86ff01cdf2ef4eefa993146f0d35ddbd89c7932eb8e5f31988a6c546b62792","schema_version":"1.0","event_id":"sha256:de86ff01cdf2ef4eefa993146f0d35ddbd89c7932eb8e5f31988a6c546b62792"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/RUHEZA2OOUJ3XGZHSWP5SBD5JW/bundle.json","state_url":"https://pith.science/pith/RUHEZA2OOUJ3XGZHSWP5SBD5JW/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/RUHEZA2OOUJ3XGZHSWP5SBD5JW/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-08T07:18:31Z","links":{"resolver":"https://pith.science/pith/RUHEZA2OOUJ3XGZHSWP5SBD5JW","bundle":"https://pith.science/pith/RUHEZA2OOUJ3XGZHSWP5SBD5JW/bundle.json","state":"https://pith.science/pith/RUHEZA2OOUJ3XGZHSWP5SBD5JW/state.json","well_known_bundle":"https://pith.science/.well-known/pith/RUHEZA2OOUJ3XGZHSWP5SBD5JW/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:RUHEZA2OOUJ3XGZHSWP5SBD5JW","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"df3932b17b7a92e5e79a49548f24f2f492c4e4a49bed9f99d77648a4f9810152","cross_cats_sorted":["cs.AI","cs.CR","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-09T17:53:06Z","title_canon_sha256":"eed0ed41615fb11ba8cb4a8fc8c1906fd302006012639b7ac71f828a6785c8ac"},"schema_version":"1.0","source":{"id":"2410.07137","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.07137","created_at":"2026-07-05T10:22:06Z"},{"alias_kind":"arxiv_version","alias_value":"2410.07137v2","created_at":"2026-07-05T10:22:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.07137","created_at":"2026-07-05T10:22:06Z"},{"alias_kind":"pith_short_12","alias_value":"RUHEZA2OOUJ3","created_at":"2026-07-05T10:22:06Z"},{"alias_kind":"pith_short_16","alias_value":"RUHEZA2OOUJ3XGZH","created_at":"2026-07-05T10:22:06Z"},{"alias_kind":"pith_short_8","alias_value":"RUHEZA2O","created_at":"2026-07-05T10:22:06Z"}],"graph_snapshots":[{"event_id":"sha256:de86ff01cdf2ef4eefa993146f0d35ddbd89c7932eb8e5f31988a6c546b62792","target":"graph","created_at":"2026-07-05T10:22:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.07137/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Automatic LLM benchmarks, such as AlpacaEval 2.0, Arena-Hard-Auto, and MT-Bench, have become popular for evaluating language models due to their cost-effectiveness and scalability compared to human evaluation. Achieving high win rates on these benchmarks can significantly boost the promotional impact of newly released language models. This promotional benefit may motivate tricks, such as manipulating model output length or style to game win rates, even though several mechanisms have been developed to control length and disentangle style to reduce gameability. Nonetheless, we show that even a \"","authors_text":"Chao Du, Jing Jiang, Min Lin, Qian Liu, Tianyu Pang, Xiaosen Zheng","cross_cats":["cs.AI","cs.CR","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-09T17:53:06Z","title":"Cheating Automatic LLM Benchmarks: Null Models Achieve High Win Rates"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.07137","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:bb666fe9901dc8bf22f1fba4df005275b8c99b4a3ed16d196fc3dfd81402a3c7","target":"record","created_at":"2026-07-05T10:22:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"df3932b17b7a92e5e79a49548f24f2f492c4e4a49bed9f99d77648a4f9810152","cross_cats_sorted":["cs.AI","cs.CR","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-09T17:53:06Z","title_canon_sha256":"eed0ed41615fb11ba8cb4a8fc8c1906fd302006012639b7ac71f828a6785c8ac"},"schema_version":"1.0","source":{"id":"2410.07137","kind":"arxiv","version":2}},"canonical_sha256":"8d0e4c834e7513bb9b27959fd9047d4d9cdc2c5adbaf49736aa271b8900650de","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"8d0e4c834e7513bb9b27959fd9047d4d9cdc2c5adbaf49736aa271b8900650de","first_computed_at":"2026-07-05T10:22:06.995158Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:22:06.995158Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"fx9I7gy3k0qQsacnDPoZZlpCAIoKIrmxrzq4VSPp0xGPiC+K89ooHcaHrjTGl8PApjTjhFq5bXZAxcrNgzw+Cg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:22:06.995710Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.07137","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:bb666fe9901dc8bf22f1fba4df005275b8c99b4a3ed16d196fc3dfd81402a3c7","sha256:de86ff01cdf2ef4eefa993146f0d35ddbd89c7932eb8e5f31988a6c546b62792"],"state_sha256":"ae0dd8467e6199aa5eda61d80425225d0d7759b0acc4b8f58ed35812de80e477"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"UiGTVUOT63ytiMz444k1gY/bP3nvIdcE26pwWYoQbmE3LYpyqu2F4B/ZwkK7k37lhJ5Q6hv28T9ABmfwsPsPDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-08T07:18:31.006265Z","bundle_sha256":"17f1079dba1dc826068c2bbcff1577b1dddfa3d1fc2a89ee05dc0e1bfd1887f8"}}