{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:N5SJRJ633PXJSRJJEA2HBB2YRI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9efa449856bab1e01dcc8557c235939aaede93cdf05f47659eaa87325d6579b1","cross_cats_sorted":["cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-31T07:48:44Z","title_canon_sha256":"69a5230988277acada53766581f74147c701447c920171f9d55d67f37799677f"},"schema_version":"1.0","source":{"id":"2410.23703","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.23703","created_at":"2026-07-05T09:29:04Z"},{"alias_kind":"arxiv_version","alias_value":"2410.23703v1","created_at":"2026-07-05T09:29:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.23703","created_at":"2026-07-05T09:29:04Z"},{"alias_kind":"pith_short_12","alias_value":"N5SJRJ633PXJ","created_at":"2026-07-05T09:29:04Z"},{"alias_kind":"pith_short_16","alias_value":"N5SJRJ633PXJSRJJ","created_at":"2026-07-05T09:29:04Z"},{"alias_kind":"pith_short_8","alias_value":"N5SJRJ63","created_at":"2026-07-05T09:29:04Z"}],"graph_snapshots":[{"event_id":"sha256:d9096f034db0d898321cf65ded2e69cf7e3a6ec8453185bf4d8383a36aefbde0","target":"graph","created_at":"2026-07-05T09:29:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.23703/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Offline evaluation of LLMs is crucial in understanding their capacities, though current methods remain underexplored in existing research. In this work, we focus on the offline evaluation of the chain-of-thought capabilities and show how to optimize LLMs based on the proposed evaluation method. To enable offline feedback with rich knowledge and reasoning paths, we use knowledge graphs (e.g., Wikidata5m) to provide feedback on the generated chain of thoughts. Due to the heterogeneity between LLM reasoning and KG structures, direct interaction and feedback from KGs on LLM behavior are challengin","authors_text":"Branislav Kveton, Jianing Wang, Jingbo Shang, Julian McAuley, Junda Wu, Lina Yao, Ruoyu Wang, Tong Yu, Xiang Chen, Xintong Li, Yu Xia, Yuxin Xiong","cross_cats":["cs.CL"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-31T07:48:44Z","title":"OCEAN: Offline Chain-of-thought Evaluation and Alignment in Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.23703","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:2c0ce5177958b0aedab3009e557328a0437d9f89840451de37c150e1082fe29b","target":"record","created_at":"2026-07-05T09:29:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9efa449856bab1e01dcc8557c235939aaede93cdf05f47659eaa87325d6579b1","cross_cats_sorted":["cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-10-31T07:48:44Z","title_canon_sha256":"69a5230988277acada53766581f74147c701447c920171f9d55d67f37799677f"},"schema_version":"1.0","source":{"id":"2410.23703","kind":"arxiv","version":1}},"canonical_sha256":"6f6498a7dbdbee99452920347087588a33bee8d44d782519cde64e5a5c06ad73","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6f6498a7dbdbee99452920347087588a33bee8d44d782519cde64e5a5c06ad73","first_computed_at":"2026-07-05T09:29:04.150441Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:29:04.150441Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"nivebKESiljQxudW9qnSQF9kFNu9yCNGkbN+9wakKYfew6C5lhF/bTzSiYZSzqu3JJLE37mNhJVXkpwL4BewCg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:29:04.150932Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.23703","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:2c0ce5177958b0aedab3009e557328a0437d9f89840451de37c150e1082fe29b","sha256:d9096f034db0d898321cf65ded2e69cf7e3a6ec8453185bf4d8383a36aefbde0"],"state_sha256":"b1ed3d4f1da54637c79d8849a84097cdc546e3a17d1b52039c434ed6adbbe5c7"}