{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:4UJ4DGFII7LUPVCRIBRL55X3F6","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"1a35559ac0ab74691d1804465c32cdbb38c95063356f8b96f325371238e4cf6f","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-04-10T15:01:46Z","title_canon_sha256":"95228dce548b1e43a5cca17fe1de4008a71ecb8631a421d3b0b1c2f3c5dd01a3"},"schema_version":"1.0","source":{"id":"2504.07825","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2504.07825","created_at":"2026-07-05T10:47:15Z"},{"alias_kind":"arxiv_version","alias_value":"2504.07825v1","created_at":"2026-07-05T10:47:15Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.07825","created_at":"2026-07-05T10:47:15Z"},{"alias_kind":"pith_short_12","alias_value":"4UJ4DGFII7LU","created_at":"2026-07-05T10:47:15Z"},{"alias_kind":"pith_short_16","alias_value":"4UJ4DGFII7LUPVCR","created_at":"2026-07-05T10:47:15Z"},{"alias_kind":"pith_short_8","alias_value":"4UJ4DGFI","created_at":"2026-07-05T10:47:15Z"}],"graph_snapshots":[{"event_id":"sha256:249dca773b58969863b0fa5f9e73465c237abe7aa5cf5f202ffe11ad73540cca","target":"graph","created_at":"2026-07-05T10:47:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2504.07825/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Common-sense reasoning is a key language model capability because it encapsulates not just specific factual knowledge but rather general language and world understanding. Measuring common-sense reasoning, therefore, is crucial for language models of different sizes and applications. One of the most widely used benchmarks for evaluating such capabilities is HellaSwag; however, in this paper, we show that it has severe construct validity issues. These issues range from basic ungrammaticality and numerous typos to misleading prompts or equally correct options. Furthermore, we show that if models ","authors_text":"Ivan P. Yamshchikov, Mattia Nee, Pavel Chizhov, Pierre-Carl Langlais","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-04-10T15:01:46Z","title":"What the HellaSwag? On the Validity of Common-Sense Reasoning Benchmarks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.07825","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1d423955b0168e5c456fd6056b6f3e91cbcd1bf30d0f964977852a6574f402cb","target":"record","created_at":"2026-07-05T10:47:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"1a35559ac0ab74691d1804465c32cdbb38c95063356f8b96f325371238e4cf6f","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-04-10T15:01:46Z","title_canon_sha256":"95228dce548b1e43a5cca17fe1de4008a71ecb8631a421d3b0b1c2f3c5dd01a3"},"schema_version":"1.0","source":{"id":"2504.07825","kind":"arxiv","version":1}},"canonical_sha256":"e513c198a847d747d4514062bef6fb2f8763fe838132b21e1fb7ef9bd6fed424","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"e513c198a847d747d4514062bef6fb2f8763fe838132b21e1fb7ef9bd6fed424","first_computed_at":"2026-07-05T10:47:15.432084Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:47:15.432084Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"i47F/PQYSo32Lh/TAKlp8SscgYAPpqYx4FbGexYDgn/v9Z/A2KLyx2Hc3Uj7/uaS12WzIGo64g4KTaU5IxvrBw==","signature_status":"signed_v1","signed_at":"2026-07-05T10:47:15.432575Z","signed_message":"canonical_sha256_bytes"},"source_id":"2504.07825","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1d423955b0168e5c456fd6056b6f3e91cbcd1bf30d0f964977852a6574f402cb","sha256:249dca773b58969863b0fa5f9e73465c237abe7aa5cf5f202ffe11ad73540cca"],"state_sha256":"b341bcfa148d8e0171eed739c1e2edbd46ae45a7382690f52f713bf603a9d7eb"}