{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:RYEXYZATZ5W3SA7WJNWWEY35VB","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"64a683d7bfb1b06765960ed4e003bfde8d5acb3c2c3dbdf43275882ed665a154","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-01T15:06:51Z","title_canon_sha256":"04689f5351a3d998d68d47d464221c3a6081560eafc310472b64403200c44876"},"schema_version":"1.0","source":{"id":"2310.01448","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.01448","created_at":"2026-07-05T07:02:32Z"},{"alias_kind":"arxiv_version","alias_value":"2310.01448v2","created_at":"2026-07-05T07:02:32Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.01448","created_at":"2026-07-05T07:02:32Z"},{"alias_kind":"pith_short_12","alias_value":"RYEXYZATZ5W3","created_at":"2026-07-05T07:02:32Z"},{"alias_kind":"pith_short_16","alias_value":"RYEXYZATZ5W3SA7W","created_at":"2026-07-05T07:02:32Z"},{"alias_kind":"pith_short_8","alias_value":"RYEXYZAT","created_at":"2026-07-05T07:02:32Z"}],"graph_snapshots":[{"event_id":"sha256:efe5c72fc9437ad3d1d6b87ff85a8dde725f6e6d2644eab3cefd53ea558656e1","target":"graph","created_at":"2026-07-05T07:02:32Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2310.01448/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Do large language models (LLMs) genuinely understand the semantics of the language, or just memorize the training data? The recent concern on potential data contamination of LLMs has raised awareness of the community to conduct research on LLMs evaluation. In this paper, we propose MSTemp, an approach that creates meta semantic templates to evaluate the semantic understanding ability of LLMs. The core of MSTemp is not to perform evaluation directly on existing benchmark datasets, but to generate new out-of-distribution (OOD) evaluation sets using existing datasets as seeds. Specifically, for a","authors_text":"Jindong Wang, Liang Chen, Qiaozhu Mei, Xing Xie, Yachuan Liu","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-01T15:06:51Z","title":"Meta Semantic Template for Evaluation of Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.01448","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e7de6748c5d081fe436bfb96dd94809f795eb12ccdc81097708cbead2a65afcf","target":"record","created_at":"2026-07-05T07:02:32Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"64a683d7bfb1b06765960ed4e003bfde8d5acb3c2c3dbdf43275882ed665a154","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-01T15:06:51Z","title_canon_sha256":"04689f5351a3d998d68d47d464221c3a6081560eafc310472b64403200c44876"},"schema_version":"1.0","source":{"id":"2310.01448","kind":"arxiv","version":2}},"canonical_sha256":"8e097c6413cf6db903f64b6d62637da85b399ee40df0bf392b4fca4cf7a30e06","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"8e097c6413cf6db903f64b6d62637da85b399ee40df0bf392b4fca4cf7a30e06","first_computed_at":"2026-07-05T07:02:32.769042Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:02:32.769042Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"k21c4ywD8iza6pPl+3WBFfJhYWAjH0aN80x4YYEU8hElQ036Edw8FRe/3Bq5qnt3RUHZuFELTAcgODR9XtWeAw==","signature_status":"signed_v1","signed_at":"2026-07-05T07:02:32.769539Z","signed_message":"canonical_sha256_bytes"},"source_id":"2310.01448","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e7de6748c5d081fe436bfb96dd94809f795eb12ccdc81097708cbead2a65afcf","sha256:efe5c72fc9437ad3d1d6b87ff85a8dde725f6e6d2644eab3cefd53ea558656e1"],"state_sha256":"740af8dd422df48bbef86980e07cdc881314a3657a00efd0f79053c58ec35779"}