{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:UOPXHLLL6527GAPXT2ST5RFEIR","short_pith_number":"pith:UOPXHLLL","canonical_record":{"source":{"id":"2012.00955","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-12-02T03:53:13Z","cross_cats_sorted":[],"title_canon_sha256":"6a99a49b4f74d5d1c87792efd6572e3e23cfe14baa55fd0981e7a457f9493843","abstract_canon_sha256":"65b30254828565582bf27549e2292cf4c67c2de02ed8a1aa7cdaa805035bd11f"},"schema_version":"1.0"},"canonical_sha256":"a39f73ad6bf775f301f79ea53ec4a44469e399b8928ca1a43fd6c5bb7cfc5551","source":{"kind":"arxiv","id":"2012.00955","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2012.00955","created_at":"2026-07-05T02:41:49Z"},{"alias_kind":"arxiv_version","alias_value":"2012.00955v2","created_at":"2026-07-05T02:41:49Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2012.00955","created_at":"2026-07-05T02:41:49Z"},{"alias_kind":"pith_short_12","alias_value":"UOPXHLLL6527","created_at":"2026-07-05T02:41:49Z"},{"alias_kind":"pith_short_16","alias_value":"UOPXHLLL6527GAPX","created_at":"2026-07-05T02:41:49Z"},{"alias_kind":"pith_short_8","alias_value":"UOPXHLLL","created_at":"2026-07-05T02:41:49Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:UOPXHLLL6527GAPXT2ST5RFEIR","target":"record","payload":{"canonical_record":{"source":{"id":"2012.00955","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-12-02T03:53:13Z","cross_cats_sorted":[],"title_canon_sha256":"6a99a49b4f74d5d1c87792efd6572e3e23cfe14baa55fd0981e7a457f9493843","abstract_canon_sha256":"65b30254828565582bf27549e2292cf4c67c2de02ed8a1aa7cdaa805035bd11f"},"schema_version":"1.0"},"canonical_sha256":"a39f73ad6bf775f301f79ea53ec4a44469e399b8928ca1a43fd6c5bb7cfc5551","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:41:49.262438Z","signature_b64":"RCyK5tVqDmZA4KxXt8Bck+lmVpvAlbQEZkBW2I1w0rTMZWSqmz4Xn6EAMMUuglUBU7JdAz1i4ZIcPtfQyGriBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a39f73ad6bf775f301f79ea53ec4a44469e399b8928ca1a43fd6c5bb7cfc5551","last_reissued_at":"2026-07-05T02:41:49.261979Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:41:49.261979Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2012.00955","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:41:49Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"DpwblwIj1w8Q2Z802MXNj/l9mJoYQGrJFhqGnxwFOOxXM3QvNMdekEPY58IjOE7BJQxqBP++0Gm/HAoM9fkWBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T12:23:21.737461Z"},"content_sha256":"0748c1488a48d6ce041affcbb9b1bfd74dd40cbe01d61f388401f8f8a1dd4a12","schema_version":"1.0","event_id":"sha256:0748c1488a48d6ce041affcbb9b1bfd74dd40cbe01d61f388401f8f8a1dd4a12"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:UOPXHLLL6527GAPXT2ST5RFEIR","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"How Can We Know When Language Models Know? On the Calibration of Language Models for Question Answering","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Graham Neubig, Haibo Ding, Jun Araki, Zhengbao Jiang","submitted_at":"2020-12-02T03:53:13Z","abstract_excerpt":"Recent works have shown that language models (LM) capture different types of knowledge regarding facts or common sense. However, because no model is perfect, they still fail to provide appropriate answers in many cases. In this paper, we ask the question \"how can we know when language models know, with confidence, the answer to a particular query?\" We examine this question from the point of view of calibration, the property of a probabilistic model's predicted probabilities actually being well correlated with the probabilities of correctness. We examine three strong generative models -- T5, BA"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2012.00955","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2012.00955/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:41:49Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"srG0kAyQJaTDAXhIVoO+E3ulX6pkxWJwmLlv/Ueh4l5fcYrafcha9R549CPNDwl4CvKlNBfnSYcLjmRc4S1gBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T12:23:21.737903Z"},"content_sha256":"7710b45e2b4ad4317f3f7a7621265fff18df7b8463eec88b0c03ad7ecb75f8d4","schema_version":"1.0","event_id":"sha256:7710b45e2b4ad4317f3f7a7621265fff18df7b8463eec88b0c03ad7ecb75f8d4"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/UOPXHLLL6527GAPXT2ST5RFEIR/bundle.json","state_url":"https://pith.science/pith/UOPXHLLL6527GAPXT2ST5RFEIR/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/UOPXHLLL6527GAPXT2ST5RFEIR/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T12:23:21Z","links":{"resolver":"https://pith.science/pith/UOPXHLLL6527GAPXT2ST5RFEIR","bundle":"https://pith.science/pith/UOPXHLLL6527GAPXT2ST5RFEIR/bundle.json","state":"https://pith.science/pith/UOPXHLLL6527GAPXT2ST5RFEIR/state.json","well_known_bundle":"https://pith.science/.well-known/pith/UOPXHLLL6527GAPXT2ST5RFEIR/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:UOPXHLLL6527GAPXT2ST5RFEIR","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"65b30254828565582bf27549e2292cf4c67c2de02ed8a1aa7cdaa805035bd11f","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-12-02T03:53:13Z","title_canon_sha256":"6a99a49b4f74d5d1c87792efd6572e3e23cfe14baa55fd0981e7a457f9493843"},"schema_version":"1.0","source":{"id":"2012.00955","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2012.00955","created_at":"2026-07-05T02:41:49Z"},{"alias_kind":"arxiv_version","alias_value":"2012.00955v2","created_at":"2026-07-05T02:41:49Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2012.00955","created_at":"2026-07-05T02:41:49Z"},{"alias_kind":"pith_short_12","alias_value":"UOPXHLLL6527","created_at":"2026-07-05T02:41:49Z"},{"alias_kind":"pith_short_16","alias_value":"UOPXHLLL6527GAPX","created_at":"2026-07-05T02:41:49Z"},{"alias_kind":"pith_short_8","alias_value":"UOPXHLLL","created_at":"2026-07-05T02:41:49Z"}],"graph_snapshots":[{"event_id":"sha256:7710b45e2b4ad4317f3f7a7621265fff18df7b8463eec88b0c03ad7ecb75f8d4","target":"graph","created_at":"2026-07-05T02:41:49Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2012.00955/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent works have shown that language models (LM) capture different types of knowledge regarding facts or common sense. However, because no model is perfect, they still fail to provide appropriate answers in many cases. In this paper, we ask the question \"how can we know when language models know, with confidence, the answer to a particular query?\" We examine this question from the point of view of calibration, the property of a probabilistic model's predicted probabilities actually being well correlated with the probabilities of correctness. We examine three strong generative models -- T5, BA","authors_text":"Graham Neubig, Haibo Ding, Jun Araki, Zhengbao Jiang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-12-02T03:53:13Z","title":"How Can We Know When Language Models Know? On the Calibration of Language Models for Question Answering"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2012.00955","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:0748c1488a48d6ce041affcbb9b1bfd74dd40cbe01d61f388401f8f8a1dd4a12","target":"record","created_at":"2026-07-05T02:41:49Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"65b30254828565582bf27549e2292cf4c67c2de02ed8a1aa7cdaa805035bd11f","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2020-12-02T03:53:13Z","title_canon_sha256":"6a99a49b4f74d5d1c87792efd6572e3e23cfe14baa55fd0981e7a457f9493843"},"schema_version":"1.0","source":{"id":"2012.00955","kind":"arxiv","version":2}},"canonical_sha256":"a39f73ad6bf775f301f79ea53ec4a44469e399b8928ca1a43fd6c5bb7cfc5551","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a39f73ad6bf775f301f79ea53ec4a44469e399b8928ca1a43fd6c5bb7cfc5551","first_computed_at":"2026-07-05T02:41:49.261979Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:41:49.261979Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"RCyK5tVqDmZA4KxXt8Bck+lmVpvAlbQEZkBW2I1w0rTMZWSqmz4Xn6EAMMUuglUBU7JdAz1i4ZIcPtfQyGriBw==","signature_status":"signed_v1","signed_at":"2026-07-05T02:41:49.262438Z","signed_message":"canonical_sha256_bytes"},"source_id":"2012.00955","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:0748c1488a48d6ce041affcbb9b1bfd74dd40cbe01d61f388401f8f8a1dd4a12","sha256:7710b45e2b4ad4317f3f7a7621265fff18df7b8463eec88b0c03ad7ecb75f8d4"],"state_sha256":"bea5e998293bf0de10c77430cd28f82b6b3705b17837804f7bda5ee456b802ac"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"k/7oEd8RF4lEGWmuStGlRUzYdxpEVymIygjVMWxQmGre0A4Ky/acAYF2IysPG0iEf9EdQ+UzviQeUtJzCM63AA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T12:23:21.739969Z","bundle_sha256":"ef6b0d0ce2ca97cd5c56aae074e6c8d35f1d0514f5764a35b712db2f3a69020e"}}