{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2019:DHWXGEHRSBXZD2QCSKCVREGPZV","short_pith_number":"pith:DHWXGEHR","canonical_record":{"source":{"id":"1904.05929","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-04-11T19:04:57Z","cross_cats_sorted":[],"title_canon_sha256":"990f0f68d071370d37b3a28f5f4c206eb02fa15401514997fbe970b3eb3cbd3e","abstract_canon_sha256":"dc498a8cf270518227a6daceae8265ea1e7bd3db329e57924543086360c1a6cc"},"schema_version":"1.0"},"canonical_sha256":"19ed7310f1906f91ea0292855890cfcd5c406fdcba219581bc3665564c8f8f4e","source":{"kind":"arxiv","id":"1904.05929","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1904.05929","created_at":"2026-05-17T23:48:44Z"},{"alias_kind":"arxiv_version","alias_value":"1904.05929v1","created_at":"2026-05-17T23:48:44Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1904.05929","created_at":"2026-05-17T23:48:44Z"},{"alias_kind":"pith_short_12","alias_value":"DHWXGEHRSBXZ","created_at":"2026-05-18T12:33:15Z"},{"alias_kind":"pith_short_16","alias_value":"DHWXGEHRSBXZD2QC","created_at":"2026-05-18T12:33:15Z"},{"alias_kind":"pith_short_8","alias_value":"DHWXGEHR","created_at":"2026-05-18T12:33:15Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2019:DHWXGEHRSBXZD2QCSKCVREGPZV","target":"record","payload":{"canonical_record":{"source":{"id":"1904.05929","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-04-11T19:04:57Z","cross_cats_sorted":[],"title_canon_sha256":"990f0f68d071370d37b3a28f5f4c206eb02fa15401514997fbe970b3eb3cbd3e","abstract_canon_sha256":"dc498a8cf270518227a6daceae8265ea1e7bd3db329e57924543086360c1a6cc"},"schema_version":"1.0"},"canonical_sha256":"19ed7310f1906f91ea0292855890cfcd5c406fdcba219581bc3665564c8f8f4e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:48:44.707981Z","signature_b64":"/MUgJifkTAraz9NEJ4KDtlhhqpz5EgxpYo1kItRJ8bwaSpA9Ue+ofP57IujRUuhGC7li19ZQdIjCPAGdKgjhBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"19ed7310f1906f91ea0292855890cfcd5c406fdcba219581bc3665564c8f8f4e","last_reissued_at":"2026-05-17T23:48:44.707455Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:48:44.707455Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1904.05929","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:48:44Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rLARjQm6kNHptecaG2LuhTLllc0I/hV7N2jEGJGnCC8FAA280NDt2HuDdDpx5I4EYPac0pqViWC9l6JUlXSwAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-03T11:02:01.545561Z"},"content_sha256":"f9cf24ba6bbb98d423d8a0f6afda4a4652e2f35d3c4da2d3fe216b9e576caee7","schema_version":"1.0","event_id":"sha256:f9cf24ba6bbb98d423d8a0f6afda4a4652e2f35d3c4da2d3fe216b9e576caee7"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2019:DHWXGEHRSBXZD2QCSKCVREGPZV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Crowdsourcing Lightweight Pyramids for Manual Summary Evaluation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"David Gabay, Hadar Ronen, Ido Dagan, Mohit Bansal, Ori Shapira, Ramakanth Pasunuru, Yael Amsterdamer, Yang Gao","submitted_at":"2019-04-11T19:04:57Z","abstract_excerpt":"Conducting a manual evaluation is considered an essential part of summary evaluation methodology. Traditionally, the Pyramid protocol, which exhaustively compares system summaries to references, has been perceived as very reliable, providing objective scores. Yet, due to the high cost of the Pyramid method and the required expertise, researchers resorted to cheaper and less thorough manual evaluation methods, such as Responsiveness and pairwise comparison, attainable via crowdsourcing. We revisit the Pyramid approach, proposing a lightweight sampling-based version that is crowdsourcable. We an"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1904.05929","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:48:44Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CJfALN1cfFOU96e5FehhbkCtTaDTpeS++hPDND+f3ETXqpIz0X+l+QdmREcSfJW8BAvupW/eGrN4vjWbxd1qAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-03T11:02:01.545912Z"},"content_sha256":"3af3b7f4c326df989df6871687ee485528404ce76dec114792a32f9d3b05305c","schema_version":"1.0","event_id":"sha256:3af3b7f4c326df989df6871687ee485528404ce76dec114792a32f9d3b05305c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/DHWXGEHRSBXZD2QCSKCVREGPZV/bundle.json","state_url":"https://pith.science/pith/DHWXGEHRSBXZD2QCSKCVREGPZV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/DHWXGEHRSBXZD2QCSKCVREGPZV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-03T11:02:01Z","links":{"resolver":"https://pith.science/pith/DHWXGEHRSBXZD2QCSKCVREGPZV","bundle":"https://pith.science/pith/DHWXGEHRSBXZD2QCSKCVREGPZV/bundle.json","state":"https://pith.science/pith/DHWXGEHRSBXZD2QCSKCVREGPZV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/DHWXGEHRSBXZD2QCSKCVREGPZV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2019:DHWXGEHRSBXZD2QCSKCVREGPZV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"dc498a8cf270518227a6daceae8265ea1e7bd3db329e57924543086360c1a6cc","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-04-11T19:04:57Z","title_canon_sha256":"990f0f68d071370d37b3a28f5f4c206eb02fa15401514997fbe970b3eb3cbd3e"},"schema_version":"1.0","source":{"id":"1904.05929","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1904.05929","created_at":"2026-05-17T23:48:44Z"},{"alias_kind":"arxiv_version","alias_value":"1904.05929v1","created_at":"2026-05-17T23:48:44Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1904.05929","created_at":"2026-05-17T23:48:44Z"},{"alias_kind":"pith_short_12","alias_value":"DHWXGEHRSBXZ","created_at":"2026-05-18T12:33:15Z"},{"alias_kind":"pith_short_16","alias_value":"DHWXGEHRSBXZD2QC","created_at":"2026-05-18T12:33:15Z"},{"alias_kind":"pith_short_8","alias_value":"DHWXGEHR","created_at":"2026-05-18T12:33:15Z"}],"graph_snapshots":[{"event_id":"sha256:3af3b7f4c326df989df6871687ee485528404ce76dec114792a32f9d3b05305c","target":"graph","created_at":"2026-05-17T23:48:44Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Conducting a manual evaluation is considered an essential part of summary evaluation methodology. Traditionally, the Pyramid protocol, which exhaustively compares system summaries to references, has been perceived as very reliable, providing objective scores. Yet, due to the high cost of the Pyramid method and the required expertise, researchers resorted to cheaper and less thorough manual evaluation methods, such as Responsiveness and pairwise comparison, attainable via crowdsourcing. We revisit the Pyramid approach, proposing a lightweight sampling-based version that is crowdsourcable. We an","authors_text":"David Gabay, Hadar Ronen, Ido Dagan, Mohit Bansal, Ori Shapira, Ramakanth Pasunuru, Yael Amsterdamer, Yang Gao","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-04-11T19:04:57Z","title":"Crowdsourcing Lightweight Pyramids for Manual Summary Evaluation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1904.05929","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:f9cf24ba6bbb98d423d8a0f6afda4a4652e2f35d3c4da2d3fe216b9e576caee7","target":"record","created_at":"2026-05-17T23:48:44Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"dc498a8cf270518227a6daceae8265ea1e7bd3db329e57924543086360c1a6cc","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-04-11T19:04:57Z","title_canon_sha256":"990f0f68d071370d37b3a28f5f4c206eb02fa15401514997fbe970b3eb3cbd3e"},"schema_version":"1.0","source":{"id":"1904.05929","kind":"arxiv","version":1}},"canonical_sha256":"19ed7310f1906f91ea0292855890cfcd5c406fdcba219581bc3665564c8f8f4e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"19ed7310f1906f91ea0292855890cfcd5c406fdcba219581bc3665564c8f8f4e","first_computed_at":"2026-05-17T23:48:44.707455Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:48:44.707455Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"/MUgJifkTAraz9NEJ4KDtlhhqpz5EgxpYo1kItRJ8bwaSpA9Ue+ofP57IujRUuhGC7li19ZQdIjCPAGdKgjhBw==","signature_status":"signed_v1","signed_at":"2026-05-17T23:48:44.707981Z","signed_message":"canonical_sha256_bytes"},"source_id":"1904.05929","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:f9cf24ba6bbb98d423d8a0f6afda4a4652e2f35d3c4da2d3fe216b9e576caee7","sha256:3af3b7f4c326df989df6871687ee485528404ce76dec114792a32f9d3b05305c"],"state_sha256":"d8ba2b6e00b1f6fca80c0bce4816e528e77543c89b788836cf00ef611055409f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"UBIjmz7emMP3i0LkwZabudH5FqLgq4UooXRJR+isX7qR+OVSwZTuI2LZ4gZ94doMYG56TmxdA1dL4AozQH0bBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-03T11:02:01.547902Z","bundle_sha256":"29e84d37cbec8d9bbbf90b05d2bd971d0f2c1c2804e9092dc9bc515c5813a4f2"}}