{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:2SI2JMM3FXEMN7WVQ3H45ERB25","short_pith_number":"pith:2SI2JMM3","schema_version":"1.0","canonical_sha256":"d491a4b19b2dc8c6fed586cfce9221d75204883f3df9f2f78c76e20031ba4ab0","source":{"kind":"arxiv","id":"2406.15468","version":2},"attestation_state":"computed","paper":{"title":"MMLU-SR: A Benchmark for Stress-Testing Reasoning Capability of Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Hao Wang, Jacob Feldman, Lazaros Gallos, Paul Kantor, Sarthak Jain, Wentian Wang","submitted_at":"2024-06-15T05:35:47Z","abstract_excerpt":"We propose MMLU-SR, a novel dataset designed to measure the true comprehension abilities of Large Language Models (LLMs) by challenging their performance in question-answering tasks with modified terms. We reasoned that an agent that \"truly\" understands a concept can still evaluate it when key terms are replaced by suitably defined alternate terms, and sought to differentiate such comprehension from mere text replacement. In our study, we modified standardized test questions by replacing a key term with a dummy word along with its definition. The key term could be in the context of questions, "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2406.15468","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-06-15T05:35:47Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"24e37ba99282d6ae0908e8e65ee31247a230d57110e50af827f296d263b337ac","abstract_canon_sha256":"6d25cef365e8ae52b95ad739ad06c3af379c493ccb533e2655e50f4d4845605d"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:15:34.234420Z","signature_b64":"r6aYqZfkGHxVVl9/MOjTFAxJOiDvCNiA/6IjNGPQXFPlwDLsf7m8IFG0pdcka0C9uV7tbHPXuLQh3PPgdwkEBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d491a4b19b2dc8c6fed586cfce9221d75204883f3df9f2f78c76e20031ba4ab0","last_reissued_at":"2026-07-05T09:15:34.233867Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:15:34.233867Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"MMLU-SR: A Benchmark for Stress-Testing Reasoning Capability of Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Hao Wang, Jacob Feldman, Lazaros Gallos, Paul Kantor, Sarthak Jain, Wentian Wang","submitted_at":"2024-06-15T05:35:47Z","abstract_excerpt":"We propose MMLU-SR, a novel dataset designed to measure the true comprehension abilities of Large Language Models (LLMs) by challenging their performance in question-answering tasks with modified terms. We reasoned that an agent that \"truly\" understands a concept can still evaluate it when key terms are replaced by suitably defined alternate terms, and sought to differentiate such comprehension from mere text replacement. In our study, we modified standardized test questions by replacing a key term with a dummy word along with its definition. The key term could be in the context of questions, "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.15468","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.15468/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2406.15468","created_at":"2026-07-05T09:15:34.233933+00:00"},{"alias_kind":"arxiv_version","alias_value":"2406.15468v2","created_at":"2026-07-05T09:15:34.233933+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.15468","created_at":"2026-07-05T09:15:34.233933+00:00"},{"alias_kind":"pith_short_12","alias_value":"2SI2JMM3FXEM","created_at":"2026-07-05T09:15:34.233933+00:00"},{"alias_kind":"pith_short_16","alias_value":"2SI2JMM3FXEMN7WV","created_at":"2026-07-05T09:15:34.233933+00:00"},{"alias_kind":"pith_short_8","alias_value":"2SI2JMM3","created_at":"2026-07-05T09:15:34.233933+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2504.18574","citing_title":"Understanding the Skill Gap in Recurrent Language Models: The Role of the Gather-and-Aggregate Mechanism","ref_index":47,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/2SI2JMM3FXEMN7WVQ3H45ERB25","json":"https://pith.science/pith/2SI2JMM3FXEMN7WVQ3H45ERB25.json","graph_json":"https://pith.science/api/pith-number/2SI2JMM3FXEMN7WVQ3H45ERB25/graph.json","events_json":"https://pith.science/api/pith-number/2SI2JMM3FXEMN7WVQ3H45ERB25/events.json","paper":"https://pith.science/paper/2SI2JMM3"},"agent_actions":{"view_html":"https://pith.science/pith/2SI2JMM3FXEMN7WVQ3H45ERB25","download_json":"https://pith.science/pith/2SI2JMM3FXEMN7WVQ3H45ERB25.json","view_paper":"https://pith.science/paper/2SI2JMM3","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2406.15468&json=true","fetch_graph":"https://pith.science/api/pith-number/2SI2JMM3FXEMN7WVQ3H45ERB25/graph.json","fetch_events":"https://pith.science/api/pith-number/2SI2JMM3FXEMN7WVQ3H45ERB25/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/2SI2JMM3FXEMN7WVQ3H45ERB25/action/timestamp_anchor","attest_storage":"https://pith.science/pith/2SI2JMM3FXEMN7WVQ3H45ERB25/action/storage_attestation","attest_author":"https://pith.science/pith/2SI2JMM3FXEMN7WVQ3H45ERB25/action/author_attestation","sign_citation":"https://pith.science/pith/2SI2JMM3FXEMN7WVQ3H45ERB25/action/citation_signature","submit_replication":"https://pith.science/pith/2SI2JMM3FXEMN7WVQ3H45ERB25/action/replication_record"}},"created_at":"2026-07-05T09:15:34.233933+00:00","updated_at":"2026-07-05T09:15:34.233933+00:00"}