{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:VP27KNMXUIV2MXBGREJDWAI6LS","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"81c8c3eded50cf64a632bb2646a3251cafde8bfd837cde8f2803a900762c286f","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"stat.ML","submitted_at":"2024-04-06T17:33:37Z","title_canon_sha256":"ddad926a62335bf250f1fe139c3b9e9b352d38f38fe3316dc5d0cfdb0ef2749a"},"schema_version":"1.0","source":{"id":"2404.04689","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.04689","created_at":"2026-07-05T08:05:40Z"},{"alias_kind":"arxiv_version","alias_value":"2404.04689v1","created_at":"2026-07-05T08:05:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.04689","created_at":"2026-07-05T08:05:40Z"},{"alias_kind":"pith_short_12","alias_value":"VP27KNMXUIV2","created_at":"2026-07-05T08:05:40Z"},{"alias_kind":"pith_short_16","alias_value":"VP27KNMXUIV2MXBG","created_at":"2026-07-05T08:05:40Z"},{"alias_kind":"pith_short_8","alias_value":"VP27KNMX","created_at":"2026-07-05T08:05:40Z"}],"graph_snapshots":[{"event_id":"sha256:47d25a13f6632d51757c952fdac0073eb9db948785fb31d2467bd4e165fdf7ce","target":"graph","created_at":"2026-07-05T08:05:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2404.04689/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"This paper proposes the use of \"multicalibration\" to yield interpretable and reliable confidence scores for outputs generated by large language models (LLMs). Multicalibration asks for calibration not just marginally, but simultaneously across various intersecting groupings of the data. We show how to form groupings for prompt/completion pairs that are correlated with the probability of correctness via two techniques: clustering within an embedding space, and \"self-annotation\" - querying the LLM by asking it various yes-or-no questions about the prompt. We also develop novel variants of multic","authors_text":"Aaron Roth, Gianluca Detommaso, Martin Bertran, Riccardo Fogliato","cross_cats":["cs.CL","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"stat.ML","submitted_at":"2024-04-06T17:33:37Z","title":"Multicalibration for Confidence Scoring in LLMs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.04689","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:164c7bb86bcc95dda6d2a280073ca34790f300b62674b28faaca883d01a47a1d","target":"record","created_at":"2026-07-05T08:05:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"81c8c3eded50cf64a632bb2646a3251cafde8bfd837cde8f2803a900762c286f","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"stat.ML","submitted_at":"2024-04-06T17:33:37Z","title_canon_sha256":"ddad926a62335bf250f1fe139c3b9e9b352d38f38fe3316dc5d0cfdb0ef2749a"},"schema_version":"1.0","source":{"id":"2404.04689","kind":"arxiv","version":1}},"canonical_sha256":"abf5f53597a22ba65c2689123b011e5c80bbd40884f7075971fa4860848e7e5f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"abf5f53597a22ba65c2689123b011e5c80bbd40884f7075971fa4860848e7e5f","first_computed_at":"2026-07-05T08:05:40.508023Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:05:40.508023Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"VjhXVvE/PzKAi1I/y7XUrP0FtkyNdKmCeZwdKOsRI2/fCTrLTcICRkaow2y8h8fr6stDkwiVUp62EeQmRvSOCA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:05:40.508503Z","signed_message":"canonical_sha256_bytes"},"source_id":"2404.04689","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:164c7bb86bcc95dda6d2a280073ca34790f300b62674b28faaca883d01a47a1d","sha256:47d25a13f6632d51757c952fdac0073eb9db948785fb31d2467bd4e165fdf7ce"],"state_sha256":"c96ab5f01526ad76af932673e44d5f1b694de77651dc3e73c33a8a375cdf6ff6"}