{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:FW6JTPTO72OJG2NRRYOM2AOURQ","short_pith_number":"pith:FW6JTPTO","canonical_record":{"source":{"id":"2004.13486","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2020-04-28T13:22:26Z","cross_cats_sorted":["cs.CL","cs.LG"],"title_canon_sha256":"c45bff83b0453bce363f12668e59a2f6829055b0ac7ae654db3b8a9f59c5b49f","abstract_canon_sha256":"2330d1e7b54ad27e4e93c4d8028ef5c5c019478bf979ccd01baca46553d88b9a"},"schema_version":"1.0"},"canonical_sha256":"2dbc99be6efe9c9369b18e1ccd01d48c22521debb2bbac1910b9360665313167","source":{"kind":"arxiv","id":"2004.13486","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2004.13486","created_at":"2026-07-05T00:59:00Z"},{"alias_kind":"arxiv_version","alias_value":"2004.13486v1","created_at":"2026-07-05T00:59:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2004.13486","created_at":"2026-07-05T00:59:00Z"},{"alias_kind":"pith_short_12","alias_value":"FW6JTPTO72OJ","created_at":"2026-07-05T00:59:00Z"},{"alias_kind":"pith_short_16","alias_value":"FW6JTPTO72OJG2NR","created_at":"2026-07-05T00:59:00Z"},{"alias_kind":"pith_short_8","alias_value":"FW6JTPTO","created_at":"2026-07-05T00:59:00Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:FW6JTPTO72OJG2NRRYOM2AOURQ","target":"record","payload":{"canonical_record":{"source":{"id":"2004.13486","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2020-04-28T13:22:26Z","cross_cats_sorted":["cs.CL","cs.LG"],"title_canon_sha256":"c45bff83b0453bce363f12668e59a2f6829055b0ac7ae654db3b8a9f59c5b49f","abstract_canon_sha256":"2330d1e7b54ad27e4e93c4d8028ef5c5c019478bf979ccd01baca46553d88b9a"},"schema_version":"1.0"},"canonical_sha256":"2dbc99be6efe9c9369b18e1ccd01d48c22521debb2bbac1910b9360665313167","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T00:59:00.013754Z","signature_b64":"q0LAJKO3vy9Sf9GazHehElRl85+KLx32Bj4nDHEpqVjVbC6P2lSnYKXcmfsdHCglKDEqjGWwpNhgGXwWeRgZAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2dbc99be6efe9c9369b18e1ccd01d48c22521debb2bbac1910b9360665313167","last_reissued_at":"2026-07-05T00:59:00.013395Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T00:59:00.013395Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2004.13486","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T00:59:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XuU1ZokFpfSu6h6/KYR+TC1B7bfY2L+An9Zysf8NT6BDiMolQFk3KS3BhjR6xl0oXUIcHchT1Ahn1mGAkFMqDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T05:44:48.663510Z"},"content_sha256":"072870be24e97dce4c0e487b757dacf0d41001b0b93a97923b2e28a51c9812c0","schema_version":"1.0","event_id":"sha256:072870be24e97dce4c0e487b757dacf0d41001b0b93a97923b2e28a51c9812c0"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:FW6JTPTO72OJG2NRRYOM2AOURQ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"On the Reliability of Test Collections for Evaluating Systems of Different Types","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL","cs.LG"],"primary_cat":"cs.IR","authors_text":"Bhaskar Mitra, Daniel Campos, Emine Yilmaz, Nick Craswell","submitted_at":"2020-04-28T13:22:26Z","abstract_excerpt":"As deep learning based models are increasingly being used for information retrieval (IR), a major challenge is to ensure the availability of test collections for measuring their quality. Test collections are generated based on pooling results of various retrieval systems, but until recently this did not include deep learning systems. This raises a major challenge for reusable evaluation: Since deep learning based models use external resources (e.g. word embeddings) and advanced representations as opposed to traditional methods that are mainly based on lexical similarity, they may return differ"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2004.13486","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2004.13486/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T00:59:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jTUuBYqw19YoR6vYBH//OcsZKUXojUm0eTgkPG7HDDNdDD2qinX1Di06qDku/n8eoH1xieuAEOo4fuQlOGAEDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T05:44:48.663881Z"},"content_sha256":"09ba93607d3381e6ba2ce011ce3f8118f2509e3e0f09e552ca578d2a9fda1b1c","schema_version":"1.0","event_id":"sha256:09ba93607d3381e6ba2ce011ce3f8118f2509e3e0f09e552ca578d2a9fda1b1c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/FW6JTPTO72OJG2NRRYOM2AOURQ/bundle.json","state_url":"https://pith.science/pith/FW6JTPTO72OJG2NRRYOM2AOURQ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/FW6JTPTO72OJG2NRRYOM2AOURQ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T05:44:48Z","links":{"resolver":"https://pith.science/pith/FW6JTPTO72OJG2NRRYOM2AOURQ","bundle":"https://pith.science/pith/FW6JTPTO72OJG2NRRYOM2AOURQ/bundle.json","state":"https://pith.science/pith/FW6JTPTO72OJG2NRRYOM2AOURQ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/FW6JTPTO72OJG2NRRYOM2AOURQ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:FW6JTPTO72OJG2NRRYOM2AOURQ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2330d1e7b54ad27e4e93c4d8028ef5c5c019478bf979ccd01baca46553d88b9a","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2020-04-28T13:22:26Z","title_canon_sha256":"c45bff83b0453bce363f12668e59a2f6829055b0ac7ae654db3b8a9f59c5b49f"},"schema_version":"1.0","source":{"id":"2004.13486","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2004.13486","created_at":"2026-07-05T00:59:00Z"},{"alias_kind":"arxiv_version","alias_value":"2004.13486v1","created_at":"2026-07-05T00:59:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2004.13486","created_at":"2026-07-05T00:59:00Z"},{"alias_kind":"pith_short_12","alias_value":"FW6JTPTO72OJ","created_at":"2026-07-05T00:59:00Z"},{"alias_kind":"pith_short_16","alias_value":"FW6JTPTO72OJG2NR","created_at":"2026-07-05T00:59:00Z"},{"alias_kind":"pith_short_8","alias_value":"FW6JTPTO","created_at":"2026-07-05T00:59:00Z"}],"graph_snapshots":[{"event_id":"sha256:09ba93607d3381e6ba2ce011ce3f8118f2509e3e0f09e552ca578d2a9fda1b1c","target":"graph","created_at":"2026-07-05T00:59:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2004.13486/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"As deep learning based models are increasingly being used for information retrieval (IR), a major challenge is to ensure the availability of test collections for measuring their quality. Test collections are generated based on pooling results of various retrieval systems, but until recently this did not include deep learning systems. This raises a major challenge for reusable evaluation: Since deep learning based models use external resources (e.g. word embeddings) and advanced representations as opposed to traditional methods that are mainly based on lexical similarity, they may return differ","authors_text":"Bhaskar Mitra, Daniel Campos, Emine Yilmaz, Nick Craswell","cross_cats":["cs.CL","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2020-04-28T13:22:26Z","title":"On the Reliability of Test Collections for Evaluating Systems of Different Types"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2004.13486","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:072870be24e97dce4c0e487b757dacf0d41001b0b93a97923b2e28a51c9812c0","target":"record","created_at":"2026-07-05T00:59:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2330d1e7b54ad27e4e93c4d8028ef5c5c019478bf979ccd01baca46553d88b9a","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2020-04-28T13:22:26Z","title_canon_sha256":"c45bff83b0453bce363f12668e59a2f6829055b0ac7ae654db3b8a9f59c5b49f"},"schema_version":"1.0","source":{"id":"2004.13486","kind":"arxiv","version":1}},"canonical_sha256":"2dbc99be6efe9c9369b18e1ccd01d48c22521debb2bbac1910b9360665313167","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2dbc99be6efe9c9369b18e1ccd01d48c22521debb2bbac1910b9360665313167","first_computed_at":"2026-07-05T00:59:00.013395Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T00:59:00.013395Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"q0LAJKO3vy9Sf9GazHehElRl85+KLx32Bj4nDHEpqVjVbC6P2lSnYKXcmfsdHCglKDEqjGWwpNhgGXwWeRgZAg==","signature_status":"signed_v1","signed_at":"2026-07-05T00:59:00.013754Z","signed_message":"canonical_sha256_bytes"},"source_id":"2004.13486","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:072870be24e97dce4c0e487b757dacf0d41001b0b93a97923b2e28a51c9812c0","sha256:09ba93607d3381e6ba2ce011ce3f8118f2509e3e0f09e552ca578d2a9fda1b1c"],"state_sha256":"27b39a6df8a3752c54fe7c7b6574a94dbbc1cc1ab3a4c490bc8466a2446c448c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"x6eIlYuQh8VCBm/OjDxcHsPzSBGyHBLPYI9RNjlqPhqA3DYsYidzByo35WssvalbcmLCCM86zHriHkhOrA/YDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T05:44:48.665866Z","bundle_sha256":"d870f9f311c723a8f8ee4f8d1077a4fb1ca7c36159d0cf5c7c0477486742cf31"}}