{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:BVPYYB6ZPKG26G4OAIJGBSNKIJ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5e1e6460ea8f0076e1f69dff73eea8276a0c7d496e7f5f0c0ce3a882c6a10d6a","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-12-07T03:35:39Z","title_canon_sha256":"ff1e090468b109cc447aa9b849b1867030cb54d9a7bda1e567bf62cd79299421"},"schema_version":"1.0","source":{"id":"2412.05520","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.05520","created_at":"2026-07-05T09:46:04Z"},{"alias_kind":"arxiv_version","alias_value":"2412.05520v1","created_at":"2026-07-05T09:46:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.05520","created_at":"2026-07-05T09:46:04Z"},{"alias_kind":"pith_short_12","alias_value":"BVPYYB6ZPKG2","created_at":"2026-07-05T09:46:04Z"},{"alias_kind":"pith_short_16","alias_value":"BVPYYB6ZPKG26G4O","created_at":"2026-07-05T09:46:04Z"},{"alias_kind":"pith_short_8","alias_value":"BVPYYB6Z","created_at":"2026-07-05T09:46:04Z"}],"graph_snapshots":[{"event_id":"sha256:61192bfac87c00a6cb3b983df436a845fbce90eaf2efb26ea8953b3addc0097c","target":"graph","created_at":"2026-07-05T09:46:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2412.05520/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Public AI benchmark results are widely broadcast by model developers as indicators of model quality within a growing and competitive market. However, these advertised scores do not necessarily reflect the traits of interest to those who will ultimately apply AI models. In this paper, we seek to understand if and how AI benchmarks are used to inform decision-making. Based on the analyses of interviews with 19 individuals who have used, or decided against using, benchmarks in their day-to-day work, we find that across these settings, participants use benchmarks as a signal of relative performanc","authors_text":"Allie Griffith, Amelia Hardy, Anka Reuel, Dylan M. Asmar, Kiana Jafari Meimandi, Lisa Soder, Michael S. Bernstein, Mykel J. Kochenderfer, Sanmi Koyejo","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-12-07T03:35:39Z","title":"More than Marketing? On the Information Value of AI Benchmarks for Practitioners"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.05520","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d8e0fcbcfb6d7483d90030b6490d9aa1c04d453aa9705c71e1f911a6a2f5004e","target":"record","created_at":"2026-07-05T09:46:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5e1e6460ea8f0076e1f69dff73eea8276a0c7d496e7f5f0c0ce3a882c6a10d6a","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-12-07T03:35:39Z","title_canon_sha256":"ff1e090468b109cc447aa9b849b1867030cb54d9a7bda1e567bf62cd79299421"},"schema_version":"1.0","source":{"id":"2412.05520","kind":"arxiv","version":1}},"canonical_sha256":"0d5f8c07d97a8daf1b8e021260c9aa424cc272370427bb9357d1946ee684c9fc","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0d5f8c07d97a8daf1b8e021260c9aa424cc272370427bb9357d1946ee684c9fc","first_computed_at":"2026-07-05T09:46:04.433180Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:46:04.433180Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"GFjJ1yEWBtVscfFSvmg36zB3G+c82Ye2epD2HzcY145HOJMz5Uz+rpfHmUhj7LhwctDQsAUy22zyE7ZUEoSSAg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:46:04.433594Z","signed_message":"canonical_sha256_bytes"},"source_id":"2412.05520","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d8e0fcbcfb6d7483d90030b6490d9aa1c04d453aa9705c71e1f911a6a2f5004e","sha256:61192bfac87c00a6cb3b983df436a845fbce90eaf2efb26ea8953b3addc0097c"],"state_sha256":"18d0f3d8508bf1bb6da184dd04f43c085bc8ae2f874cfb2f364deec96d2a0da3"}