{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:5JGTY6ORTJ4X6LJ6ZV3LBUPPI5","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c5210459b77ebf01180c8b886b88f4141b8ba3c8f26619ec7a373ccaec0be3e0","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.HC","submitted_at":"2022-03-08T23:31:06Z","title_canon_sha256":"271fcd8cfe7d665e8c4cd225e7e6cc535458317810765e253e4880e6208f58b9"},"schema_version":"1.0","source":{"id":"2203.04444","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2203.04444","created_at":"2026-07-05T04:03:18Z"},{"alias_kind":"arxiv_version","alias_value":"2203.04444v1","created_at":"2026-07-05T04:03:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2203.04444","created_at":"2026-07-05T04:03:18Z"},{"alias_kind":"pith_short_12","alias_value":"5JGTY6ORTJ4X","created_at":"2026-07-05T04:03:18Z"},{"alias_kind":"pith_short_16","alias_value":"5JGTY6ORTJ4X6LJ6","created_at":"2026-07-05T04:03:18Z"},{"alias_kind":"pith_short_8","alias_value":"5JGTY6OR","created_at":"2026-07-05T04:03:18Z"}],"graph_snapshots":[{"event_id":"sha256:93a386b0eaaf3c88fdb242de75365727d498a98912d52eae6977a0cd0bee7884","target":"graph","created_at":"2026-07-05T04:03:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2203.04444/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Human perceptual studies are the gold standard for the evaluation of many research tasks in machine learning, linguistics, and psychology. However, these studies require significant time and cost to perform. As a result, many researchers use objective measures that can correlate poorly with human evaluation. When subjective evaluations are performed, they are often not reported with sufficient detail to ensure reproducibility. We propose Reproducible Subjective Evaluation (ReSEval), an open-source framework for quickly deploying crowdsourced subjective evaluations directly from Python. ReSEval","authors_text":"Brian Tang, Bryan Pardo, Gefei Tan, Max Morrison","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.HC","submitted_at":"2022-03-08T23:31:06Z","title":"Reproducible Subjective Evaluation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2203.04444","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5c524a8d9f5645deb0f4a13f1717665704721107783e9c63b725ab306d040f32","target":"record","created_at":"2026-07-05T04:03:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c5210459b77ebf01180c8b886b88f4141b8ba3c8f26619ec7a373ccaec0be3e0","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.HC","submitted_at":"2022-03-08T23:31:06Z","title_canon_sha256":"271fcd8cfe7d665e8c4cd225e7e6cc535458317810765e253e4880e6208f58b9"},"schema_version":"1.0","source":{"id":"2203.04444","kind":"arxiv","version":1}},"canonical_sha256":"ea4d3c79d19a797f2d3ecd76b0d1ef47603e4e07a28ca201b9e749cf6560e05d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"ea4d3c79d19a797f2d3ecd76b0d1ef47603e4e07a28ca201b9e749cf6560e05d","first_computed_at":"2026-07-05T04:03:18.441730Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:03:18.441730Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"M0+kRGPSmTPrgZ9Txb0JdRrxI3AxeT2B1DEZjkuvjdV3sk3rPR82x4Y0um1TrKJs++YX4y8fehr1MKz5zbvZCg==","signature_status":"signed_v1","signed_at":"2026-07-05T04:03:18.442069Z","signed_message":"canonical_sha256_bytes"},"source_id":"2203.04444","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5c524a8d9f5645deb0f4a13f1717665704721107783e9c63b725ab306d040f32","sha256:93a386b0eaaf3c88fdb242de75365727d498a98912d52eae6977a0cd0bee7884"],"state_sha256":"6d5e6795d14326df049b22cbbb58b59fd2ad08200c2e04ceecf24652c3e85e69"}