{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:GFLXO3LQROXL4GK6JLRZRSZ2WI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ba865dd57e95f114555e3721e5aeb9d30f99fa45ebe4ea255c407560887d9836","cross_cats_sorted":["cs.CL","cs.LG","q-fin.ST","stat.AP"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ML","submitted_at":"2024-12-10T21:57:25Z","title_canon_sha256":"b8a8fd299f7b02c7a0be722dc73685d0abd009cd197686e8edd2cd549ff3089c"},"schema_version":"1.0","source":{"id":"2412.12148","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.12148","created_at":"2026-07-05T09:50:10Z"},{"alias_kind":"arxiv_version","alias_value":"2412.12148v1","created_at":"2026-07-05T09:50:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.12148","created_at":"2026-07-05T09:50:10Z"},{"alias_kind":"pith_short_12","alias_value":"GFLXO3LQROXL","created_at":"2026-07-05T09:50:10Z"},{"alias_kind":"pith_short_16","alias_value":"GFLXO3LQROXL4GK6","created_at":"2026-07-05T09:50:10Z"},{"alias_kind":"pith_short_8","alias_value":"GFLXO3LQ","created_at":"2026-07-05T09:50:10Z"}],"graph_snapshots":[{"event_id":"sha256:5a7d393b4caabf6c52cc461af0224c46b87eaeb32e00c54615e2660b960c592c","target":"graph","created_at":"2026-07-05T09:50:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2412.12148/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"To ensure and monitor large language models (LLMs) reliably, various evaluation metrics have been proposed in the literature. However, there is little research on prescribing a methodology to identify a robust threshold on these metrics even though there are many serious implications of an incorrect choice of the thresholds during deployment of the LLMs. Translating the traditional model risk management (MRM) guidelines within regulated industries such as the financial industry, we propose a step-by-step recipe for picking a threshold for a given LLM evaluation metric. We emphasize that such a","authors_text":"Bhaskarjit Sarmah, Dhagash Mehta, Jingrao Lyu, Mingshu Li, Nathalia Castellanos, Sebastian Frank, Stefano Pasquali","cross_cats":["cs.CL","cs.LG","q-fin.ST","stat.AP"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ML","submitted_at":"2024-12-10T21:57:25Z","title":"How to Choose a Threshold for an Evaluation Metric for Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.12148","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:3713a13861d2c41d220b1ba9ef6fe9725a59bc01bf3960b4c6892ecd3740fa3a","target":"record","created_at":"2026-07-05T09:50:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ba865dd57e95f114555e3721e5aeb9d30f99fa45ebe4ea255c407560887d9836","cross_cats_sorted":["cs.CL","cs.LG","q-fin.ST","stat.AP"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ML","submitted_at":"2024-12-10T21:57:25Z","title_canon_sha256":"b8a8fd299f7b02c7a0be722dc73685d0abd009cd197686e8edd2cd549ff3089c"},"schema_version":"1.0","source":{"id":"2412.12148","kind":"arxiv","version":1}},"canonical_sha256":"3157776d708baebe195e4ae398cb3ab222cf5b313838c1bdc21849bd5cb9450e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3157776d708baebe195e4ae398cb3ab222cf5b313838c1bdc21849bd5cb9450e","first_computed_at":"2026-07-05T09:50:10.560758Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:50:10.560758Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"FOIpsUDIHTaHjkggZlj0FNdYJeZFfv0yn5Z1TItJ9tUqaQOlYpA/Z/YnQvcswpy+f9vdC1Y6aOaWvS64tnQWCA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:50:10.561208Z","signed_message":"canonical_sha256_bytes"},"source_id":"2412.12148","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:3713a13861d2c41d220b1ba9ef6fe9725a59bc01bf3960b4c6892ecd3740fa3a","sha256:5a7d393b4caabf6c52cc461af0224c46b87eaeb32e00c54615e2660b960c592c"],"state_sha256":"a250a91f583317859942109062b5d331c9364ea8e2b03d78a7890430bf25f73d"}