{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:DNOH7Z4VIRN2ROBCHEG7QK3DZZ","short_pith_number":"pith:DNOH7Z4V","canonical_record":{"source":{"id":"2310.17623","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-26T17:43:13Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"bfb6075958a2222fd528d6e7601181c575b4dd27817ca99b546c09ab8b0199a8","abstract_canon_sha256":"a49d972e73213b96fa8d8cd8e5c8bb76227565a919f31c42cd681c72acb19a77"},"schema_version":"1.0"},"canonical_sha256":"1b5c7fe795445ba8b822390df82b63ce7818f17d8f6cf5fe70728b402a95440f","source":{"kind":"arxiv","id":"2310.17623","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.17623","created_at":"2026-07-05T07:16:01Z"},{"alias_kind":"arxiv_version","alias_value":"2310.17623v2","created_at":"2026-07-05T07:16:01Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.17623","created_at":"2026-07-05T07:16:01Z"},{"alias_kind":"pith_short_12","alias_value":"DNOH7Z4VIRN2","created_at":"2026-07-05T07:16:01Z"},{"alias_kind":"pith_short_16","alias_value":"DNOH7Z4VIRN2ROBC","created_at":"2026-07-05T07:16:01Z"},{"alias_kind":"pith_short_8","alias_value":"DNOH7Z4V","created_at":"2026-07-05T07:16:01Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:DNOH7Z4VIRN2ROBCHEG7QK3DZZ","target":"record","payload":{"canonical_record":{"source":{"id":"2310.17623","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-26T17:43:13Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"bfb6075958a2222fd528d6e7601181c575b4dd27817ca99b546c09ab8b0199a8","abstract_canon_sha256":"a49d972e73213b96fa8d8cd8e5c8bb76227565a919f31c42cd681c72acb19a77"},"schema_version":"1.0"},"canonical_sha256":"1b5c7fe795445ba8b822390df82b63ce7818f17d8f6cf5fe70728b402a95440f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:16:01.906935Z","signature_b64":"xzzrXfXhpkWdPZ1iySVSeurM4CQv4wTV0OBE6cZ1lekixd1Mu1dI6fl6vLCsaEnqz1ESxV+aWY1wNEOOgbUlDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1b5c7fe795445ba8b822390df82b63ce7818f17d8f6cf5fe70728b402a95440f","last_reissued_at":"2026-07-05T07:16:01.906416Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:16:01.906416Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2310.17623","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:16:01Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+72dHtCmA1iDZ+pXFgrnEZ4Qx1Q/RuM5+gtVUiCGs+1lU8SsKyaSG6y5l2sUlAxkQG/OwZJL4+W+hxPJeGKsAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T07:41:28.859384Z"},"content_sha256":"af17db4aaf993e820f08cb37d1a2b5d565a2f5bb03dfffe8708b4734156719a5","schema_version":"1.0","event_id":"sha256:af17db4aaf993e820f08cb37d1a2b5d565a2f5bb03dfffe8708b4734156719a5"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:DNOH7Z4VIRN2ROBCHEG7QK3DZZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Proving Test Set Contamination in Black Box Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Faisal Ladhak, Nicole Meister, Niladri Chatterji, Tatsunori B. Hashimoto, Yonatan Oren","submitted_at":"2023-10-26T17:43:13Z","abstract_excerpt":"Large language models are trained on vast amounts of internet data, prompting concerns and speculation that they have memorized public benchmarks. Going from speculation to proof of contamination is challenging, as the pretraining data used by proprietary models are often not publicly accessible. We show that it is possible to provide provable guarantees of test set contamination in language models without access to pretraining data or model weights. Our approach leverages the fact that when there is no data contamination, all orderings of an exchangeable benchmark should be equally likely. In"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.17623","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.17623/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:16:01Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"MNvHC3YG9r5fhARPVMQXM+BM/Y9jRHmC9TNQEbIQ3MgVMQb25D92LbPVC5i1QEPkP5HUdQCPsvc9rjoXDwt9DA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T07:41:28.863015Z"},"content_sha256":"aa0b03ac9d587ddd8ba53ef51bd1d6c2903f223ce830cd727963f09290c1da44","schema_version":"1.0","event_id":"sha256:aa0b03ac9d587ddd8ba53ef51bd1d6c2903f223ce830cd727963f09290c1da44"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/DNOH7Z4VIRN2ROBCHEG7QK3DZZ/bundle.json","state_url":"https://pith.science/pith/DNOH7Z4VIRN2ROBCHEG7QK3DZZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/DNOH7Z4VIRN2ROBCHEG7QK3DZZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-18T07:41:28Z","links":{"resolver":"https://pith.science/pith/DNOH7Z4VIRN2ROBCHEG7QK3DZZ","bundle":"https://pith.science/pith/DNOH7Z4VIRN2ROBCHEG7QK3DZZ/bundle.json","state":"https://pith.science/pith/DNOH7Z4VIRN2ROBCHEG7QK3DZZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/DNOH7Z4VIRN2ROBCHEG7QK3DZZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:DNOH7Z4VIRN2ROBCHEG7QK3DZZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a49d972e73213b96fa8d8cd8e5c8bb76227565a919f31c42cd681c72acb19a77","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-26T17:43:13Z","title_canon_sha256":"bfb6075958a2222fd528d6e7601181c575b4dd27817ca99b546c09ab8b0199a8"},"schema_version":"1.0","source":{"id":"2310.17623","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.17623","created_at":"2026-07-05T07:16:01Z"},{"alias_kind":"arxiv_version","alias_value":"2310.17623v2","created_at":"2026-07-05T07:16:01Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.17623","created_at":"2026-07-05T07:16:01Z"},{"alias_kind":"pith_short_12","alias_value":"DNOH7Z4VIRN2","created_at":"2026-07-05T07:16:01Z"},{"alias_kind":"pith_short_16","alias_value":"DNOH7Z4VIRN2ROBC","created_at":"2026-07-05T07:16:01Z"},{"alias_kind":"pith_short_8","alias_value":"DNOH7Z4V","created_at":"2026-07-05T07:16:01Z"}],"graph_snapshots":[{"event_id":"sha256:aa0b03ac9d587ddd8ba53ef51bd1d6c2903f223ce830cd727963f09290c1da44","target":"graph","created_at":"2026-07-05T07:16:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2310.17623/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models are trained on vast amounts of internet data, prompting concerns and speculation that they have memorized public benchmarks. Going from speculation to proof of contamination is challenging, as the pretraining data used by proprietary models are often not publicly accessible. We show that it is possible to provide provable guarantees of test set contamination in language models without access to pretraining data or model weights. Our approach leverages the fact that when there is no data contamination, all orderings of an exchangeable benchmark should be equally likely. In","authors_text":"Faisal Ladhak, Nicole Meister, Niladri Chatterji, Tatsunori B. Hashimoto, Yonatan Oren","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-26T17:43:13Z","title":"Proving Test Set Contamination in Black Box Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.17623","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:af17db4aaf993e820f08cb37d1a2b5d565a2f5bb03dfffe8708b4734156719a5","target":"record","created_at":"2026-07-05T07:16:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a49d972e73213b96fa8d8cd8e5c8bb76227565a919f31c42cd681c72acb19a77","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-26T17:43:13Z","title_canon_sha256":"bfb6075958a2222fd528d6e7601181c575b4dd27817ca99b546c09ab8b0199a8"},"schema_version":"1.0","source":{"id":"2310.17623","kind":"arxiv","version":2}},"canonical_sha256":"1b5c7fe795445ba8b822390df82b63ce7818f17d8f6cf5fe70728b402a95440f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"1b5c7fe795445ba8b822390df82b63ce7818f17d8f6cf5fe70728b402a95440f","first_computed_at":"2026-07-05T07:16:01.906416Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:16:01.906416Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"xzzrXfXhpkWdPZ1iySVSeurM4CQv4wTV0OBE6cZ1lekixd1Mu1dI6fl6vLCsaEnqz1ESxV+aWY1wNEOOgbUlDA==","signature_status":"signed_v1","signed_at":"2026-07-05T07:16:01.906935Z","signed_message":"canonical_sha256_bytes"},"source_id":"2310.17623","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:af17db4aaf993e820f08cb37d1a2b5d565a2f5bb03dfffe8708b4734156719a5","sha256:aa0b03ac9d587ddd8ba53ef51bd1d6c2903f223ce830cd727963f09290c1da44"],"state_sha256":"0dd3813a45ba98cfa2a7af62de41a8afc38b9bac0228a143e34c8dc360773a4d"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rXRHICEqt1XUu4IQ1cFWejDeHczDprdklhE8oIZFOFSnjWuoovl1VzbHBMW0FrJ7xqoveIFSrS8BK3ArzahDAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-18T07:41:28.868989Z","bundle_sha256":"52984d0de5fe29ecf7fec571e09a522e23ea12353eb7a06b7d3fc7ca66d29375"}}