{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:7T3DMJIJHQSQZS75SAVRFWPET5","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"6146da6e7309ac1dfe86c0d7a0d0998667186a197050ded528650f5a91a12b6a","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2021-05-20T02:21:27Z","title_canon_sha256":"29ce547aaf4afdf3127eb74b015cf2cf315da872fd3961801116345057b6eb85"},"schema_version":"1.0","source":{"id":"2105.09457","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2105.09457","created_at":"2026-07-05T03:31:27Z"},{"alias_kind":"arxiv_version","alias_value":"2105.09457v1","created_at":"2026-07-05T03:31:27Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2105.09457","created_at":"2026-07-05T03:31:27Z"},{"alias_kind":"pith_short_12","alias_value":"7T3DMJIJHQSQ","created_at":"2026-07-05T03:31:27Z"},{"alias_kind":"pith_short_16","alias_value":"7T3DMJIJHQSQZS75","created_at":"2026-07-05T03:31:27Z"},{"alias_kind":"pith_short_8","alias_value":"7T3DMJIJ","created_at":"2026-07-05T03:31:27Z"}],"graph_snapshots":[{"event_id":"sha256:aac5dce60509813d87a9ce119df86500249b0abca1c1d7a5e569f412b71ff054","target":"graph","created_at":"2026-07-05T03:31:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2105.09457/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We consider a class of variable effort human annotation tasks in which the number of labels required per item can greatly vary (e.g., finding all faces in an image, named entities in a text, bird calls in an audio recording, etc.). In such tasks, some items require far more effort than others to annotate. Furthermore, the per-item annotation effort is not known until after each item is annotated since determining the number of labels required is an implicit part of the annotation task itself. On an image bounding-box task with crowdsourced annotators, we show that annotator accuracy and recall","authors_text":"Danula Hettiachchi, Matthew Lease, Mike Schaekermann, Tristan McKinney","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2021-05-20T02:21:27Z","title":"The Challenge of Variable Effort Crowdsourcing and How Visible Gold Can Help"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2105.09457","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:88f86a71d5324d8e61ea44fbb0d32b6118030eb96886ea3382fe43cb1ccb1cd5","target":"record","created_at":"2026-07-05T03:31:27Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"6146da6e7309ac1dfe86c0d7a0d0998667186a197050ded528650f5a91a12b6a","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2021-05-20T02:21:27Z","title_canon_sha256":"29ce547aaf4afdf3127eb74b015cf2cf315da872fd3961801116345057b6eb85"},"schema_version":"1.0","source":{"id":"2105.09457","kind":"arxiv","version":1}},"canonical_sha256":"fcf63625093c250ccbfd902b12d9e49f7ca6f1f7b55f51b919e9e91245e54dc7","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"fcf63625093c250ccbfd902b12d9e49f7ca6f1f7b55f51b919e9e91245e54dc7","first_computed_at":"2026-07-05T03:31:27.333771Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T03:31:27.333771Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"B7zO9Fs5F5J4TzKJlNPdBK55WaBi83Vmaz6RvAQAnH7TMZuvWXsi+NVOigTOLJJQFx8u+0hsanBiSWYvUF6WBg==","signature_status":"signed_v1","signed_at":"2026-07-05T03:31:27.334332Z","signed_message":"canonical_sha256_bytes"},"source_id":"2105.09457","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:88f86a71d5324d8e61ea44fbb0d32b6118030eb96886ea3382fe43cb1ccb1cd5","sha256:aac5dce60509813d87a9ce119df86500249b0abca1c1d7a5e569f412b71ff054"],"state_sha256":"7f9a8aed59b61dc4874e9f47901ab4c6536c0c6ce816e1ee38f88c7499735a91"}