{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2022:5JGTY6ORTJ4X6LJ6ZV3LBUPPI5","short_pith_number":"pith:5JGTY6OR","schema_version":"1.0","canonical_sha256":"ea4d3c79d19a797f2d3ecd76b0d1ef47603e4e07a28ca201b9e749cf6560e05d","source":{"kind":"arxiv","id":"2203.04444","version":1},"attestation_state":"computed","paper":{"title":"Reproducible Subjective Evaluation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.HC","authors_text":"Brian Tang, Bryan Pardo, Gefei Tan, Max Morrison","submitted_at":"2022-03-08T23:31:06Z","abstract_excerpt":"Human perceptual studies are the gold standard for the evaluation of many research tasks in machine learning, linguistics, and psychology. However, these studies require significant time and cost to perform. As a result, many researchers use objective measures that can correlate poorly with human evaluation. When subjective evaluations are performed, they are often not reported with sufficient detail to ensure reproducibility. We propose Reproducible Subjective Evaluation (ReSEval), an open-source framework for quickly deploying crowdsourced subjective evaluations directly from Python. ReSEval"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2203.04444","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.HC","submitted_at":"2022-03-08T23:31:06Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"271fcd8cfe7d665e8c4cd225e7e6cc535458317810765e253e4880e6208f58b9","abstract_canon_sha256":"c5210459b77ebf01180c8b886b88f4141b8ba3c8f26619ec7a373ccaec0be3e0"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:03:18.442069Z","signature_b64":"M0+kRGPSmTPrgZ9Txb0JdRrxI3AxeT2B1DEZjkuvjdV3sk3rPR82x4Y0um1TrKJs++YX4y8fehr1MKz5zbvZCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ea4d3c79d19a797f2d3ecd76b0d1ef47603e4e07a28ca201b9e749cf6560e05d","last_reissued_at":"2026-07-05T04:03:18.441730Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:03:18.441730Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Reproducible Subjective Evaluation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.HC","authors_text":"Brian Tang, Bryan Pardo, Gefei Tan, Max Morrison","submitted_at":"2022-03-08T23:31:06Z","abstract_excerpt":"Human perceptual studies are the gold standard for the evaluation of many research tasks in machine learning, linguistics, and psychology. However, these studies require significant time and cost to perform. As a result, many researchers use objective measures that can correlate poorly with human evaluation. When subjective evaluations are performed, they are often not reported with sufficient detail to ensure reproducibility. We propose Reproducible Subjective Evaluation (ReSEval), an open-source framework for quickly deploying crowdsourced subjective evaluations directly from Python. ReSEval"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2203.04444","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2203.04444/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2203.04444","created_at":"2026-07-05T04:03:18.441785+00:00"},{"alias_kind":"arxiv_version","alias_value":"2203.04444v1","created_at":"2026-07-05T04:03:18.441785+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2203.04444","created_at":"2026-07-05T04:03:18.441785+00:00"},{"alias_kind":"pith_short_12","alias_value":"5JGTY6ORTJ4X","created_at":"2026-07-05T04:03:18.441785+00:00"},{"alias_kind":"pith_short_16","alias_value":"5JGTY6ORTJ4X6LJ6","created_at":"2026-07-05T04:03:18.441785+00:00"},{"alias_kind":"pith_short_8","alias_value":"5JGTY6OR","created_at":"2026-07-05T04:03:18.441785+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2506.00950","citing_title":"Crowdsourcing MUSHRA Tests in the Age of Generative Speech Technologies: A Comparative Analysis of Subjective and Objective Testing Methods","ref_index":21,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/5JGTY6ORTJ4X6LJ6ZV3LBUPPI5","json":"https://pith.science/pith/5JGTY6ORTJ4X6LJ6ZV3LBUPPI5.json","graph_json":"https://pith.science/api/pith-number/5JGTY6ORTJ4X6LJ6ZV3LBUPPI5/graph.json","events_json":"https://pith.science/api/pith-number/5JGTY6ORTJ4X6LJ6ZV3LBUPPI5/events.json","paper":"https://pith.science/paper/5JGTY6OR"},"agent_actions":{"view_html":"https://pith.science/pith/5JGTY6ORTJ4X6LJ6ZV3LBUPPI5","download_json":"https://pith.science/pith/5JGTY6ORTJ4X6LJ6ZV3LBUPPI5.json","view_paper":"https://pith.science/paper/5JGTY6OR","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2203.04444&json=true","fetch_graph":"https://pith.science/api/pith-number/5JGTY6ORTJ4X6LJ6ZV3LBUPPI5/graph.json","fetch_events":"https://pith.science/api/pith-number/5JGTY6ORTJ4X6LJ6ZV3LBUPPI5/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/5JGTY6ORTJ4X6LJ6ZV3LBUPPI5/action/timestamp_anchor","attest_storage":"https://pith.science/pith/5JGTY6ORTJ4X6LJ6ZV3LBUPPI5/action/storage_attestation","attest_author":"https://pith.science/pith/5JGTY6ORTJ4X6LJ6ZV3LBUPPI5/action/author_attestation","sign_citation":"https://pith.science/pith/5JGTY6ORTJ4X6LJ6ZV3LBUPPI5/action/citation_signature","submit_replication":"https://pith.science/pith/5JGTY6ORTJ4X6LJ6ZV3LBUPPI5/action/replication_record"}},"created_at":"2026-07-05T04:03:18.441785+00:00","updated_at":"2026-07-05T04:03:18.441785+00:00"}