{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:DNEB7LQQRHGZUDVDCWTN6FV73Q","short_pith_number":"pith:DNEB7LQQ","schema_version":"1.0","canonical_sha256":"1b481fae1089cd9a0ea315a6df16bfdc383aaea2b7d74e7bdbaa70d89238f336","source":{"kind":"arxiv","id":"2306.06918","version":2},"attestation_state":"computed","paper":{"title":"The Devil is in the Details: On the Pitfalls of Event Extraction Evaluation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Feng Yao, Hao Peng, Juanzi Li, Kaisheng Zeng, Lei Hou, Weixing Shen, Xiaozhi Wang, Zhiyuan Liu","submitted_at":"2023-06-12T07:38:31Z","abstract_excerpt":"Event extraction (EE) is a crucial task aiming at extracting events from texts, which includes two subtasks: event detection (ED) and event argument extraction (EAE). In this paper, we check the reliability of EE evaluations and identify three major pitfalls: (1) The data preprocessing discrepancy makes the evaluation results on the same dataset not directly comparable, but the data preprocessing details are not widely noted and specified in papers. (2) The output space discrepancy of different model paradigms makes different-paradigm EE models lack grounds for comparison and also leads to unc"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2306.06918","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-06-12T07:38:31Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"4c83212a7e930bbaac1e8bbc5829c5d6615793371956ba13c4b0cd4de0d59669","abstract_canon_sha256":"0598066df35dbe23a19948be426841fa4fea7e82995ca2ed5bae64a667113b8d"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:20:45.332719Z","signature_b64":"JfQfgh3XT1OmQcjVy0r/NYTWkeYt1CqFsg8DWYEmuY//YMOtZjNxcSpkUC9VPM+P2lHWkt4LV9ArV0P3FjfkDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1b481fae1089cd9a0ea315a6df16bfdc383aaea2b7d74e7bdbaa70d89238f336","last_reissued_at":"2026-07-05T06:20:45.332286Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:20:45.332286Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"The Devil is in the Details: On the Pitfalls of Event Extraction Evaluation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Feng Yao, Hao Peng, Juanzi Li, Kaisheng Zeng, Lei Hou, Weixing Shen, Xiaozhi Wang, Zhiyuan Liu","submitted_at":"2023-06-12T07:38:31Z","abstract_excerpt":"Event extraction (EE) is a crucial task aiming at extracting events from texts, which includes two subtasks: event detection (ED) and event argument extraction (EAE). In this paper, we check the reliability of EE evaluations and identify three major pitfalls: (1) The data preprocessing discrepancy makes the evaluation results on the same dataset not directly comparable, but the data preprocessing details are not widely noted and specified in papers. (2) The output space discrepancy of different model paradigms makes different-paradigm EE models lack grounds for comparison and also leads to unc"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2306.06918","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2306.06918/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2306.06918","created_at":"2026-07-05T06:20:45.332344+00:00"},{"alias_kind":"arxiv_version","alias_value":"2306.06918v2","created_at":"2026-07-05T06:20:45.332344+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2306.06918","created_at":"2026-07-05T06:20:45.332344+00:00"},{"alias_kind":"pith_short_12","alias_value":"DNEB7LQQRHGZ","created_at":"2026-07-05T06:20:45.332344+00:00"},{"alias_kind":"pith_short_16","alias_value":"DNEB7LQQRHGZUDVD","created_at":"2026-07-05T06:20:45.332344+00:00"},{"alias_kind":"pith_short_8","alias_value":"DNEB7LQQ","created_at":"2026-07-05T06:20:45.332344+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/DNEB7LQQRHGZUDVDCWTN6FV73Q","json":"https://pith.science/pith/DNEB7LQQRHGZUDVDCWTN6FV73Q.json","graph_json":"https://pith.science/api/pith-number/DNEB7LQQRHGZUDVDCWTN6FV73Q/graph.json","events_json":"https://pith.science/api/pith-number/DNEB7LQQRHGZUDVDCWTN6FV73Q/events.json","paper":"https://pith.science/paper/DNEB7LQQ"},"agent_actions":{"view_html":"https://pith.science/pith/DNEB7LQQRHGZUDVDCWTN6FV73Q","download_json":"https://pith.science/pith/DNEB7LQQRHGZUDVDCWTN6FV73Q.json","view_paper":"https://pith.science/paper/DNEB7LQQ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2306.06918&json=true","fetch_graph":"https://pith.science/api/pith-number/DNEB7LQQRHGZUDVDCWTN6FV73Q/graph.json","fetch_events":"https://pith.science/api/pith-number/DNEB7LQQRHGZUDVDCWTN6FV73Q/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/DNEB7LQQRHGZUDVDCWTN6FV73Q/action/timestamp_anchor","attest_storage":"https://pith.science/pith/DNEB7LQQRHGZUDVDCWTN6FV73Q/action/storage_attestation","attest_author":"https://pith.science/pith/DNEB7LQQRHGZUDVDCWTN6FV73Q/action/author_attestation","sign_citation":"https://pith.science/pith/DNEB7LQQRHGZUDVDCWTN6FV73Q/action/citation_signature","submit_replication":"https://pith.science/pith/DNEB7LQQRHGZUDVDCWTN6FV73Q/action/replication_record"}},"created_at":"2026-07-05T06:20:45.332344+00:00","updated_at":"2026-07-05T06:20:45.332344+00:00"}