{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:CNAOYMRK6GYPXBQXOJN65V3D4J","short_pith_number":"pith:CNAOYMRK","schema_version":"1.0","canonical_sha256":"1340ec322af1b0fb8617725beed763e258d944ec588159f55a82dd3d55eaecf6","source":{"kind":"arxiv","id":"2309.13233","version":1},"attestation_state":"computed","paper":{"title":"User Simulation with Large Language Models for Evaluating Task-Oriented Dialogue","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Arshit Gupta, James Gung, Raphael Shu, Saab Mansour, Salvatore Romeo, Sam Davidson, Yi Zhang","submitted_at":"2023-09-23T02:04:57Z","abstract_excerpt":"One of the major impediments to the development of new task-oriented dialogue (TOD) systems is the need for human evaluation at multiple stages and iterations of the development process. In an effort to move toward automated evaluation of TOD, we propose a novel user simulator built using recently developed large pretrained language models (LLMs). In order to increase the linguistic diversity of our system relative to the related previous work, we do not fine-tune the LLMs used by our system on existing TOD datasets; rather we use in-context learning to prompt the LLMs to generate robust and l"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2309.13233","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-23T02:04:57Z","cross_cats_sorted":[],"title_canon_sha256":"7fca9b7a38055d466b00390708a73f6947d3d9b8cceea871f4757cf530f78e09","abstract_canon_sha256":"63e293274ff7aee00f5aac271a6e62ec605f49e8b239004723bbb1b19f2cbcf6"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:53:48.995173Z","signature_b64":"catS+UinceO67SW/64EIfxQr2cCObUmto7KCPvBV9+i2TXwiq1Quf/VPrSWXuPhNrOAQXUHtSQjR9z4O7t1JDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1340ec322af1b0fb8617725beed763e258d944ec588159f55a82dd3d55eaecf6","last_reissued_at":"2026-07-05T06:53:48.994744Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:53:48.994744Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"User Simulation with Large Language Models for Evaluating Task-Oriented Dialogue","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Arshit Gupta, James Gung, Raphael Shu, Saab Mansour, Salvatore Romeo, Sam Davidson, Yi Zhang","submitted_at":"2023-09-23T02:04:57Z","abstract_excerpt":"One of the major impediments to the development of new task-oriented dialogue (TOD) systems is the need for human evaluation at multiple stages and iterations of the development process. In an effort to move toward automated evaluation of TOD, we propose a novel user simulator built using recently developed large pretrained language models (LLMs). In order to increase the linguistic diversity of our system relative to the related previous work, we do not fine-tune the LLMs used by our system on existing TOD datasets; rather we use in-context learning to prompt the LLMs to generate robust and l"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.13233","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.13233/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2309.13233","created_at":"2026-07-05T06:53:48.994798+00:00"},{"alias_kind":"arxiv_version","alias_value":"2309.13233v1","created_at":"2026-07-05T06:53:48.994798+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.13233","created_at":"2026-07-05T06:53:48.994798+00:00"},{"alias_kind":"pith_short_12","alias_value":"CNAOYMRK6GYP","created_at":"2026-07-05T06:53:48.994798+00:00"},{"alias_kind":"pith_short_16","alias_value":"CNAOYMRK6GYPXBQX","created_at":"2026-07-05T06:53:48.994798+00:00"},{"alias_kind":"pith_short_8","alias_value":"CNAOYMRK","created_at":"2026-07-05T06:53:48.994798+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":7,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2607.01557","citing_title":"DiPS: Dialogue Policy Selection for High-Stakes Persuasion Agents","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2605.20506","citing_title":"Reinforcing Human Behavior Simulation via Verbal Feedback","ref_index":90,"is_internal_anchor":false},{"citing_arxiv_id":"2605.18630","citing_title":"SCICONVBENCH: Benchmarking LLMs on Multi-Turn Clarification for Task Formulation in Computational Science","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12525","citing_title":"PERCEIVE: A Benchmark for Personalized Emotion and Communication Behavior Understanding on Social Media","ref_index":61,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08334","citing_title":"SalesSim: Benchmarking and Aligning Multimodal Language Models as Retail User Simulators","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2605.09808","citing_title":"Quantifying the Utility of User Simulators for Building Collaborative LLM Assistants","ref_index":69,"is_internal_anchor":false},{"citing_arxiv_id":"2604.13828","citing_title":"MUSE: Multi-Domain Chinese User Simulation via Self-Evolving Profiles and Rubric-Guided Alignment","ref_index":1,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/CNAOYMRK6GYPXBQXOJN65V3D4J","json":"https://pith.science/pith/CNAOYMRK6GYPXBQXOJN65V3D4J.json","graph_json":"https://pith.science/api/pith-number/CNAOYMRK6GYPXBQXOJN65V3D4J/graph.json","events_json":"https://pith.science/api/pith-number/CNAOYMRK6GYPXBQXOJN65V3D4J/events.json","paper":"https://pith.science/paper/CNAOYMRK"},"agent_actions":{"view_html":"https://pith.science/pith/CNAOYMRK6GYPXBQXOJN65V3D4J","download_json":"https://pith.science/pith/CNAOYMRK6GYPXBQXOJN65V3D4J.json","view_paper":"https://pith.science/paper/CNAOYMRK","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2309.13233&json=true","fetch_graph":"https://pith.science/api/pith-number/CNAOYMRK6GYPXBQXOJN65V3D4J/graph.json","fetch_events":"https://pith.science/api/pith-number/CNAOYMRK6GYPXBQXOJN65V3D4J/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/CNAOYMRK6GYPXBQXOJN65V3D4J/action/timestamp_anchor","attest_storage":"https://pith.science/pith/CNAOYMRK6GYPXBQXOJN65V3D4J/action/storage_attestation","attest_author":"https://pith.science/pith/CNAOYMRK6GYPXBQXOJN65V3D4J/action/author_attestation","sign_citation":"https://pith.science/pith/CNAOYMRK6GYPXBQXOJN65V3D4J/action/citation_signature","submit_replication":"https://pith.science/pith/CNAOYMRK6GYPXBQXOJN65V3D4J/action/replication_record"}},"created_at":"2026-07-05T06:53:48.994798+00:00","updated_at":"2026-07-05T06:53:48.994798+00:00"}