{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:6WDKJNYXX3PP45SP2QSCYNORB7","short_pith_number":"pith:6WDKJNYX","canonical_record":{"source":{"id":"2104.06387","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-13T17:45:50Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"d42ed6210a36bcb905cafbe79453c72d1850da765722ffe691a410db8732d5a0","abstract_canon_sha256":"199857a985fe839add2cab286a7738ddaae03a844a7f8e7d4ca0ca60a7d01442"},"schema_version":"1.0"},"canonical_sha256":"f586a4b717bedefe764fd4242c35d10ff1d86408ff9affdf2ee37752cecfbc20","source":{"kind":"arxiv","id":"2104.06387","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2104.06387","created_at":"2026-07-05T02:54:30Z"},{"alias_kind":"arxiv_version","alias_value":"2104.06387v2","created_at":"2026-07-05T02:54:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.06387","created_at":"2026-07-05T02:54:30Z"},{"alias_kind":"pith_short_12","alias_value":"6WDKJNYXX3PP","created_at":"2026-07-05T02:54:30Z"},{"alias_kind":"pith_short_16","alias_value":"6WDKJNYXX3PP45SP","created_at":"2026-07-05T02:54:30Z"},{"alias_kind":"pith_short_8","alias_value":"6WDKJNYX","created_at":"2026-07-05T02:54:30Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:6WDKJNYXX3PP45SP2QSCYNORB7","target":"record","payload":{"canonical_record":{"source":{"id":"2104.06387","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-13T17:45:50Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"d42ed6210a36bcb905cafbe79453c72d1850da765722ffe691a410db8732d5a0","abstract_canon_sha256":"199857a985fe839add2cab286a7738ddaae03a844a7f8e7d4ca0ca60a7d01442"},"schema_version":"1.0"},"canonical_sha256":"f586a4b717bedefe764fd4242c35d10ff1d86408ff9affdf2ee37752cecfbc20","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:54:30.878917Z","signature_b64":"5COBqvQqTI7OWNeYZVfrt3pSdl/02rRKMDCFOfYrqS9KLWp+OJs4JgHCeocYrmWx2QL56G6guke00RLTmeUXCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f586a4b717bedefe764fd4242c35d10ff1d86408ff9affdf2ee37752cecfbc20","last_reissued_at":"2026-07-05T02:54:30.878488Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:54:30.878488Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2104.06387","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:54:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"95YEP/+JCrZo6BuS6g5/GBm9a4Qb0YKoKeVz6j+aLd6ESuQOEWssldeACR5EkmDF4QbTZkBUTD6G88ib3v6GAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-15T11:27:53.743617Z"},"content_sha256":"846ff6c190cf2c553d851f6cefa2f00249ad05fce9a774510ed2e6bdf83f4c3f","schema_version":"1.0","event_id":"sha256:846ff6c190cf2c553d851f6cefa2f00249ad05fce9a774510ed2e6bdf83f4c3f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:6WDKJNYXX3PP45SP2QSCYNORB7","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"ExplainaBoard: An Explainable Leaderboard for NLP","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Graham Neubig, Jinlan Fu, Junqi Dai, Pengfei Liu, Shuaicheng Chang, Weizhe Yuan, Yang Xiao, Yixin Liu, Zihuiwen Ye, Zi-Yi Dou","submitted_at":"2021-04-13T17:45:50Z","abstract_excerpt":"With the rapid development of NLP research, leaderboards have emerged as one tool to track the performance of various systems on various NLP tasks. They are effective in this goal to some extent, but generally present a rather simplistic one-dimensional view of the submitted systems, communicated only through holistic accuracy numbers. In this paper, we present a new conceptualization and implementation of NLP evaluation: the ExplainaBoard, which in addition to inheriting the functionality of the standard leaderboard, also allows researchers to (i) diagnose strengths and weaknesses of a single"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.06387","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2104.06387/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:54:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"bMOirKyA2Ookw5cBbY7+dyAWNabnGzbkSo1ed5sH2QQ8ZlT9Oq1i+i70ZJWgX3Hf4X8DHsxU/oG+Uk9gSBoUBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-15T11:27:53.744148Z"},"content_sha256":"dc423a3d680f1f222c760237cf2736ea58141a7ad4b21756cf8d4870ab51c500","schema_version":"1.0","event_id":"sha256:dc423a3d680f1f222c760237cf2736ea58141a7ad4b21756cf8d4870ab51c500"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/6WDKJNYXX3PP45SP2QSCYNORB7/bundle.json","state_url":"https://pith.science/pith/6WDKJNYXX3PP45SP2QSCYNORB7/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/6WDKJNYXX3PP45SP2QSCYNORB7/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-15T11:27:53Z","links":{"resolver":"https://pith.science/pith/6WDKJNYXX3PP45SP2QSCYNORB7","bundle":"https://pith.science/pith/6WDKJNYXX3PP45SP2QSCYNORB7/bundle.json","state":"https://pith.science/pith/6WDKJNYXX3PP45SP2QSCYNORB7/state.json","well_known_bundle":"https://pith.science/.well-known/pith/6WDKJNYXX3PP45SP2QSCYNORB7/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:6WDKJNYXX3PP45SP2QSCYNORB7","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"199857a985fe839add2cab286a7738ddaae03a844a7f8e7d4ca0ca60a7d01442","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-13T17:45:50Z","title_canon_sha256":"d42ed6210a36bcb905cafbe79453c72d1850da765722ffe691a410db8732d5a0"},"schema_version":"1.0","source":{"id":"2104.06387","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2104.06387","created_at":"2026-07-05T02:54:30Z"},{"alias_kind":"arxiv_version","alias_value":"2104.06387v2","created_at":"2026-07-05T02:54:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.06387","created_at":"2026-07-05T02:54:30Z"},{"alias_kind":"pith_short_12","alias_value":"6WDKJNYXX3PP","created_at":"2026-07-05T02:54:30Z"},{"alias_kind":"pith_short_16","alias_value":"6WDKJNYXX3PP45SP","created_at":"2026-07-05T02:54:30Z"},{"alias_kind":"pith_short_8","alias_value":"6WDKJNYX","created_at":"2026-07-05T02:54:30Z"}],"graph_snapshots":[{"event_id":"sha256:dc423a3d680f1f222c760237cf2736ea58141a7ad4b21756cf8d4870ab51c500","target":"graph","created_at":"2026-07-05T02:54:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2104.06387/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"With the rapid development of NLP research, leaderboards have emerged as one tool to track the performance of various systems on various NLP tasks. They are effective in this goal to some extent, but generally present a rather simplistic one-dimensional view of the submitted systems, communicated only through holistic accuracy numbers. In this paper, we present a new conceptualization and implementation of NLP evaluation: the ExplainaBoard, which in addition to inheriting the functionality of the standard leaderboard, also allows researchers to (i) diagnose strengths and weaknesses of a single","authors_text":"Graham Neubig, Jinlan Fu, Junqi Dai, Pengfei Liu, Shuaicheng Chang, Weizhe Yuan, Yang Xiao, Yixin Liu, Zihuiwen Ye, Zi-Yi Dou","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-13T17:45:50Z","title":"ExplainaBoard: An Explainable Leaderboard for NLP"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.06387","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:846ff6c190cf2c553d851f6cefa2f00249ad05fce9a774510ed2e6bdf83f4c3f","target":"record","created_at":"2026-07-05T02:54:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"199857a985fe839add2cab286a7738ddaae03a844a7f8e7d4ca0ca60a7d01442","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-13T17:45:50Z","title_canon_sha256":"d42ed6210a36bcb905cafbe79453c72d1850da765722ffe691a410db8732d5a0"},"schema_version":"1.0","source":{"id":"2104.06387","kind":"arxiv","version":2}},"canonical_sha256":"f586a4b717bedefe764fd4242c35d10ff1d86408ff9affdf2ee37752cecfbc20","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f586a4b717bedefe764fd4242c35d10ff1d86408ff9affdf2ee37752cecfbc20","first_computed_at":"2026-07-05T02:54:30.878488Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:54:30.878488Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"5COBqvQqTI7OWNeYZVfrt3pSdl/02rRKMDCFOfYrqS9KLWp+OJs4JgHCeocYrmWx2QL56G6guke00RLTmeUXCw==","signature_status":"signed_v1","signed_at":"2026-07-05T02:54:30.878917Z","signed_message":"canonical_sha256_bytes"},"source_id":"2104.06387","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:846ff6c190cf2c553d851f6cefa2f00249ad05fce9a774510ed2e6bdf83f4c3f","sha256:dc423a3d680f1f222c760237cf2736ea58141a7ad4b21756cf8d4870ab51c500"],"state_sha256":"73c2519e544a3c18fa9e4476562375201b3c317f940cfbdf8cadb80a167da434"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"P/1ieOvB9qTC9LeuDlta5CzhbP+3r6HZc7pXgdLNPxrlowe4znJmZPyEuZSZOdlkdBP3l5zwlOVHJU+ZrO4eAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-15T11:27:53.747597Z","bundle_sha256":"9f6df71e0fcb43faff66d8904681984537b528bbc89ea7fddcc25e520abdb396"}}