{"as_of":"2026-08-17T22:09:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:01e65edbe4341b21d7bdd877d72935e2819ab28bd3b2656359173e12d3b1c16d","coverage":[{"denominator":56,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":56,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-11T11:22:56.865686Z","state":"measured"},{"denominator":58,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":58,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-17T06:30:58.91139+00:00","state":"measured"},{"denominator":2,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":2,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-05T14:55:56.666623Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-05-15T05:19:46.615078Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.15557","snapshot_observed_at":"2026-08-05T14:55:56.666623Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.20737","last_updated":"2025-08-28T13:00:28Z","snapshot_observed_at":"2026-08-16T23:09:11.422986Z","submitted_at":"2025-08-28T13:00:28Z","title":"Rethinking Testing for LLM Applications: Characteristics, Challenges, and a Lightweight Interaction Protocol","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-05T14:55:56.666623Z"},"links":{"cited_paper":"/paper/2412.15557","citing_paper":"/paper/2508.20737"},"observation_digest":"sha256:b70f6752c9291c5ace6d799637c627bebb46dc7cea89cb7e442583ce1b5cd9b4","observation_id":"74526514-5f58-4b19-a06b-67597f6aecc0","resolution":{"observed_at":"2026-08-05T14:55:56.666623Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"cited_work":{"arxiv_id":"2412.15557","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2412.15557","snapshot_observed_at":"2026-06-19T17:10:42.202763Z","title":"Mortar: Multi-turn metamorphic testing for LLM-based dialogue systems","venue":null,"work_id":"e6fe0b2e-eeeb-4f3c-80c4-776233bf117f","year":2025},"citing_paper":{"arxiv_id":"2605.13898","last_updated":"2026-05-12T13:47:26Z","snapshot_observed_at":"2026-08-10T22:06:02.817307Z","submitted_at":"2026-05-12T13:47:26Z","title":"Bidirectional Empowerment of Metamorphic Testing and Large Language Models: A Systematic Survey","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-05-15T05:15:57.846560Z"},"links":{"cited_paper":"/paper/2412.15557","citing_paper":"/paper/2605.13898"},"observation_digest":"sha256:1b0a4b2658ebb5cfc6d117aea3740d06f6abcedfa33a4721c215b35129d643b0","observation_id":"61500a84-b75f-47ec-b7ab-6ab4ed8e434e","resolution":{"observed_at":"2026-06-19T17:10:42.202763Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2412.15557/citation-record","integrity":"/paper/2412.15557/integrity","json":"/paper/2412.15557/citation-record.json","paper":"/paper/2412.15557"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.692914Z","title":"A review of dialogue systems: current trends and future directions,","venue":null,"work_id":"bf0dffe9-13c5-40dd-a570-1f8c37415bcb","year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.595184Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:992f1cc2c308ecaa9b0bca8c46d6ae3536f5e1f2fc5b3732be76e1077cb92f9d","observation_id":"cd5da5e4-d763-425d-8245-21bb7407f39c","resolution":{"observed_at":"2026-08-11T11:22:57.697758Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.678000Z","title":"Software testing of generative ai systems: Challenges and opportunities,","venue":null,"work_id":"4efc9848-219b-4f0a-a03b-1f739915f290","year":2023},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.600378Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:e78e2bc0aedcc766fb7de8f04fa02acced0eea944e943b9fac5e0c71fc3836f2","observation_id":"069dff8c-2df6-49a9-84bd-42f5311879f6","resolution":{"observed_at":"2026-08-11T11:22:57.683310Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.663291Z","title":"Hotpotqa: A dataset for diverse, explainable multi- hop question answering,","venue":null,"work_id":"4294534a-4dd1-4b7d-8f9e-ad510070f7a7","year":2018},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.605305Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:470a0a6f175a58107f5825861b1733f320a1f25ce7073561176df4c91250053d","observation_id":"6fa4f9ae-75d9-449a-9f38-3471c356838e","resolution":{"observed_at":"2026-08-11T11:22:57.668160Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.648497Z","title":"Decodingtrust: A comprehensive assessment of trustworthiness in gpt models","venue":null,"work_id":"472e7c58-526b-4bd6-b8c1-453a7cf75c88","year":2023},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.610027Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:45701abf05634b417a1e1ae6763afe4ae6b50df97abecd00ba5a60bff8fefe0b","observation_id":"1fcf0792-1540-4255-b963-2876d245627f","resolution":{"observed_at":"2026-08-11T11:22:57.653480Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2009.03300","last_updated":"2021-01-12T18:57:11Z","snapshot_observed_at":"2026-08-13T20:44:28.824685Z","submitted_at":"2020-09-07T17:59:25Z","title":"Measuring Massive Multitask Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.03300","snapshot_observed_at":"2026-08-11T11:22:56.614625Z","title":"Measuring massive multitask language understanding,","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.614625Z"},"links":{"cited_paper":"/paper/2009.03300","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:ab92555af6dc98f0e2db37326c38fffaab2f0b18f23a6895ced06ce52d8214a9","observation_id":"499566f3-2434-4cdd-9078-95b8bb5a7bcd","resolution":{"observed_at":"2026-08-11T11:22:56.614625Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.12022","last_updated":"2023-11-20T18:57:34Z","snapshot_observed_at":"2026-08-16T07:06:07.822225Z","submitted_at":"2023-11-20T18:57:34Z","title":"GPQA: A Graduate-Level Google-Proof Q&A Benchmark","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.12022","snapshot_observed_at":"2026-08-11T11:22:56.619709Z","title":"Gpqa: A graduate-level google-proof q&a benchmark,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.619709Z"},"links":{"cited_paper":"/paper/2311.12022","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:5d2ca32752af12fc421aa365c9ee264e65520c2dfee201856b9c3599520a7f98","observation_id":"bb1c302b-7e4f-4c79-a870-5c3cbcf6c383","resolution":{"observed_at":"2026-08-11T11:22:56.619709Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:56.625099Z","title":"A survey on evaluation of large language models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.625099Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:5aacefd4193f9c71a131581e89390bc4c96f44db296352b05b2d68bfc5c32ed4","observation_id":"a7f683bf-66bc-40a8-9c36-4172a54a3a27","resolution":{"observed_at":"2026-08-11T11:22:56.625099Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.624104Z","title":"ShareGPT Data,","venue":null,"work_id":"b4109100-39ef-4800-ab1d-8c1e8a7882eb","year":2023},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.629377Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:19f2de7d4f10574c6aecf4d16a8cc583a2166da0c517619a07905b03a36a7baa","observation_id":"769be06d-e7b7-47a2-95e8-08677ad9b1a5","resolution":{"observed_at":"2026-08-11T11:22:57.628738Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2408.15221","last_updated":"2024-09-04T00:58:59Z","snapshot_observed_at":"2026-08-16T13:23:25.577041Z","submitted_at":"2024-08-27T17:33:30Z","title":"LLM Defenses Are Not Robust to Multi-Turn Human Jailbreaks Yet","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.15221","snapshot_observed_at":"2026-08-11T11:22:56.634008Z","title":"Llm defenses are not robust to multi-turn human jailbreaks yet,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.634008Z"},"links":{"cited_paper":"/paper/2408.15221","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:fc72bc0b2d28cd4ecba09d50a6a08593d3a22e2c964a2960024014aa8b9212d5","observation_id":"75876efb-9200-462b-8abe-be8ed00a86b0","resolution":{"observed_at":"2026-08-11T11:22:56.634008Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.609628Z","title":"MT-bench-101: A fine-grained benchmark for evaluating large language models in multi-turn dialogues,","venue":null,"work_id":"59d44b35-1e73-47c8-9350-feb24993c0a8","year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.638711Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:749cb634fe3db4ceb9f2b8e7a6e0becee4a27b6a5e6728d3f9cc7e6de89673ff","observation_id":"122dbb91-1f5a-4e53-a094-6b990a864e5c","resolution":{"observed_at":"2026-08-11T11:22:57.614487Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:56.643140Z","title":"Judging llm-as-a-judge with mt-bench and chatbot arena,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.643140Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:5b90f9499dfe7df6fb69a103dec8950ba2bb1ea5dd087dbd680d916faf49060a","observation_id":"662f4db7-c759-4fab-bdd2-7d671afac391","resolution":{"observed_at":"2026-08-11T11:22:56.643140Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.584576Z","title":"Appli- cation of metamorphic testing in numerical analysis,","venue":null,"work_id":"b05adc2b-d28f-4246-abf0-e08a1bf67360","year":1998},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.652293Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:14aab616565520ec8e278ca00d7cfa302e1836153ac062029d8c5f29e3c52184","observation_id":"a903227b-2518-481d-9957-2412a706135e","resolution":{"observed_at":"2026-08-11T11:22:57.589805Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.570338Z","title":"Testing your question answering software via asking recursively,","venue":null,"work_id":"d8ddf1b3-3d72-410d-93c6-30fefe32f2a6","year":2021},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.656541Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:5280f6aa6bb0a130a383986e879027811fdaa6b9f2990075edb088e58fd8efe9","observation_id":"1f1d8f1d-ec60-4c42-a527-4d7a36b44144","resolution":{"observed_at":"2026-08-11T11:22:57.574992Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.541430Z","title":"Metal: Metamorphic testing framework for analyzing large-language model qualities,","venue":null,"work_id":"3da90a15-ca66-481e-ab1d-fbfbb534ff1c","year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.665077Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:cb2423033be380f7a45207dea5e48dc67c68be3179e48490c95e8126d2468e68","observation_id":"4f909497-011e-4753-9cdd-801c2411cbf4","resolution":{"observed_at":"2026-08-11T11:22:57.546199Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.525714Z","title":"Neural approaches to conversational ai,","venue":null,"work_id":"f70b83b1-3352-454a-bc8b-86121e5d4e20","year":2018},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.669258Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:ccecb6147ada1062118234de4a9119c7fbeeac52cc19e5de242b4ea858ca5fc8","observation_id":"aedc3dfa-54dd-4ef7-9ee4-4b3686601f68","resolution":{"observed_at":"2026-08-11T11:22:57.530963Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.510303Z","title":"Coqa: A conversational question answering challenge,","venue":null,"work_id":"1e22b68e-bfa6-49bc-a685-07f5eb610f52","year":2019},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.673614Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:0427ade2b70d60cec0fee2f5044d83a6f2ad9a3f2d5d07c6fdea0302c649dc2b","observation_id":"220a23ff-2156-4662-a256-d32c159d9f24","resolution":{"observed_at":"2026-08-11T11:22:57.514915Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.495736Z","title":"A survey of dialogue system evaluation,","venue":null,"work_id":"42b276aa-291a-4e61-969a-f77cef968898","year":2020},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.678228Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:137b0d34b3cb809afc1df39dee3e54ecccda37ec657eef273a9ed069493c679e","observation_id":"21103b77-a78f-42e8-98c3-2bd3807b01a7","resolution":{"observed_at":"2026-08-11T11:22:57.500603Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.19736","last_updated":"2023-11-25T17:35:12Z","snapshot_observed_at":"2026-08-17T07:14:41.768935Z","submitted_at":"2023-10-30T17:00:52Z","title":"Evaluating Large Language Models: A Comprehensive Survey","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.19736","snapshot_observed_at":"2026-08-11T11:22:56.682754Z","title":"Evaluating large language models: A comprehensive survey,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.682754Z"},"links":{"cited_paper":"/paper/2310.19736","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:d914f8dfdb0d4e32500e499f675d2019d9e875bae8af72c1b1829569167ea928","observation_id":"cebd7e80-5b9d-468f-adfc-b00a36abe7e8","resolution":{"observed_at":"2026-08-11T11:22:56.682754Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.13570","last_updated":"2020-10-01T00:00:19Z","snapshot_observed_at":"2026-08-17T16:43:47.942270Z","submitted_at":"2020-09-28T18:36:23Z","title":"DialoGLUE: A Natural Language Understanding Benchmark for Task-Oriented Dialogue","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.13570","snapshot_observed_at":"2026-08-11T11:22:56.687426Z","title":"Dialoglue: A natural language understanding benchmark for task-oriented dialogue,","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.687426Z"},"links":{"cited_paper":"/paper/2009.13570","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:ac311b47d6252afe564292847e4d5dab803da3efb0edbf83926b2bd674bcbe02","observation_id":"512b31bf-753c-402a-aa56-d99f9a4ad693","resolution":{"observed_at":"2026-08-11T11:22:56.687426Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:56.692088Z","title":"Drowzee: Metamorphic testing for fact-conflicting hallucination detection in large language models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.692088Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:9622477d77783fbd0995f24f6fe9a2c11799367a3d67919641335f2e1aa4684f","observation_id":"03b49369-5801-459d-86ed-8da74b0d23b5","resolution":{"observed_at":"2026-08-11T11:22:56.692088Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.481468Z","title":"Leveraging large language models for nlg evaluation: Advances and challenges,","venue":null,"work_id":"efd3d2a7-bd54-47f5-ae8e-fca129faa73c","year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.696695Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:bcf69c79532bed51d70e7f8d0c0c877c5a02fece749f270092648a377dfc71e8","observation_id":"07ce1f3a-6e0d-4698-93b8-0d15661efa1d","resolution":{"observed_at":"2026-08-11T11:22:57.485891Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2311.01677","last_updated":"2024-03-29T11:35:30Z","snapshot_observed_at":"2026-08-17T16:43:48.424978Z","submitted_at":"2023-11-03T02:59:56Z","title":"DialogBench: Evaluating LLMs as Human-like Dialogue Systems","version":2},"cited_work":{"arxiv_id":"2311.01677","doi":null,"metadata_source":"pith","pith_arxiv_id":"2311.01677","snapshot_observed_at":"2026-08-11T11:22:57.051727Z","title":"DialogBench: Evaluating LLMs as Human-like Dialogue Systems","venue":"cs.CL","work_id":"c966d431-0d1c-40d9-91fd-5ae53e86977e","year":2023},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.701351Z"},"links":{"cited_paper":"/paper/2311.01677","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:78bf45f4300158e0b29e6452970d22b8c61a7b49aa9a987b52412402283987b5","observation_id":"e166fa5e-c210-4f11-a55c-35b1573ba0f3","resolution":{"observed_at":"2026-08-11T11:22:57.058752Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.05229","last_updated":"2025-08-27T16:24:39Z","snapshot_observed_at":"2026-08-16T08:54:56.543625Z","submitted_at":"2024-10-07T17:36:37Z","title":"GSM-Symbolic: Understanding the Limitations of Mathematical Reasoning in Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.05229","snapshot_observed_at":"2026-08-11T11:22:56.706061Z","title":"Gsm-symbolic: Understanding the limitations of mathematical reasoning in large language models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.706061Z"},"links":{"cited_paper":"/paper/2410.05229","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:96ad797c488258ef8d8610d32ecf7d023ca19ff88338a7c8bdff015ba63c326a","observation_id":"68e838b5-9a1c-47fd-892c-7425c794e264","resolution":{"observed_at":"2026-08-11T11:22:56.706061Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.467038Z","title":"Parrot: Enhancing multi-turn instruction fol- lowing for large language models,","venue":null,"work_id":"1e4c460a-7b29-48e8-bab4-ce09aa092fff","year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.710870Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:4bfb9136ecc562e3a41e003c1ff5e4a9da9090bd58ec5ad9d84e63f3bcf2f86d","observation_id":"063417dd-fc9b-481b-a57c-626b2c45513d","resolution":{"observed_at":"2026-08-11T11:22:57.471940Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.16745","last_updated":"2024-01-30T04:50:28Z","snapshot_observed_at":"2026-08-16T14:23:23.248848Z","submitted_at":"2024-01-30T04:50:28Z","title":"MT-Eval: A Multi-Turn Capabilities Evaluation Benchmark for Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.16745","snapshot_observed_at":"2026-08-11T11:22:56.715566Z","title":"Mt-eval: A multi-turn capabili- ties evaluation benchmark for large language models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.715566Z"},"links":{"cited_paper":"/paper/2401.16745","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:7498bae39b479bb3119675eee298e2ad7e5d8a4651e1ac0d4ae8342fd4b31d99","observation_id":"83aea4ad-7624-47e5-8d3b-e0ca6f8fcdae","resolution":{"observed_at":"2026-08-11T11:22:56.715566Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.10691","last_updated":"2024-03-12T15:53:06Z","snapshot_observed_at":"2026-08-16T14:59:24.009262Z","submitted_at":"2023-09-19T15:25:42Z","title":"MINT: Evaluating LLMs in Multi-turn Interaction with Tools and Language Feedback","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.10691","snapshot_observed_at":"2026-08-11T11:22:56.721158Z","title":"Mint: Evaluating llms in multi-turn interaction with tools and language feedback,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.721158Z"},"links":{"cited_paper":"/paper/2309.10691","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:09d0a4da5b80471110bc6f2fc325da968828592a575130a87a4379ee7a54d597","observation_id":"cc51e12d-9b0c-4636-b45d-2e8181925a91","resolution":{"observed_at":"2026-08-11T11:22:56.721158Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.452294Z","title":"BotChat: Evaluating LLMs’ capabilities of having multi-turn dialogues,","venue":null,"work_id":"ac5b990d-79c2-44f5-bd5d-3735936aaf45","year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.726393Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:0ba6ac32d9f42e39c5ae362c8b729f0f32c2ec5b3c3daf297e251bfed590d500","observation_id":"7745d24c-fa75-455a-acea-86af814f6c8d","resolution":{"observed_at":"2026-08-11T11:22:57.456991Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:56.731001Z","title":"A survey on metamorphic testing,","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.731001Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:447a10d374170142507e334bd1335712ebc71573710c7fe60e114d884e9de1c0","observation_id":"65f4c591-b58d-4cd9-91d7-b4680f66caf1","resolution":{"observed_at":"2026-08-11T11:22:56.731001Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2002.12543","last_updated":"2020-02-28T04:47:31Z","snapshot_observed_at":"2026-08-14T05:31:36.259685Z","submitted_at":"2020-02-28T04:47:31Z","title":"Metamorphic Testing: A New Approach for Generating Next Test Cases","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2002.12543","snapshot_observed_at":"2026-08-11T11:22:56.735593Z","title":"Metamorphic test- ing: a new approach for generating next test cases,","venue":null,"work_id":null,"year":2002},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.735593Z"},"links":{"cited_paper":"/paper/2002.12543","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:a8bac1aa14d33a14657c813ef929bd4bded8d28e04be15948669e46d0c2a6b30","observation_id":"5efa980f-5fab-4174-8fdd-9fe24a249e5d","resolution":{"observed_at":"2026-08-11T11:22:56.735593Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:56.740449Z","title":"Metamorphic testing: A review of challenges and opportunities,","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.740449Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:a929c2bf2ec41184deee90d070709d7f5e097d4d1f049b726ba680846090c50e","observation_id":"25b83a0f-8c07-4164-a6b2-072fae49bc1f","resolution":{"observed_at":"2026-08-11T11:22:56.740449Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.418794Z","title":"Machine learning test- ing: Survey, landscapes and horizons,","venue":null,"work_id":"fceda19c-937e-48c6-af6c-de238ae4f8db","year":2020},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.744964Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:7c10bc9c8da35c542e99d54db098098d8f0669a95a4a630f65dc47dfcb6366fe","observation_id":"613f4ce7-4151-4c5d-92ff-61037b05e7f4","resolution":{"observed_at":"2026-08-11T11:22:57.423816Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.555855Z","title":"Natural test generation for precise testing of question answering software,","venue":null,"work_id":"afd7f098-f1b2-46e4-9058-116029b23069","year":2022},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.749631Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:a4156cf3caa8d4e5371dd6a4d162fc78de6903c289c50024af675c67f700f8ca","observation_id":"0b89fa87-c083-4589-8a7f-e8d7d84baf51","resolution":{"observed_at":"2026-08-11T11:22:57.560566Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.403312Z","title":"Dialtest: automated testing for recurrent- neural-network-driven dialogue systems,","venue":null,"work_id":"31e69f2a-ff8f-4c5d-b917-d912f9712046","year":2021},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.754682Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:c45796f5335aef1e0aeebc66e9de724e10cc6d664167a69d8ef79ecfae6c45e0","observation_id":"31ac894d-a33b-4956-b466-07b1d114f783","resolution":{"observed_at":"2026-08-11T11:22:57.408643Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.387768Z","title":"A metamorphic testing approach for assessing question answering systems,","venue":null,"work_id":"b2b99642-4f81-48d5-84f0-de40ec101f9a","year":2021},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.759289Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:7d41161f84b8857384a4d5923e94cc01f31bfa13bb5bdb4439d419f2aaf36ddb","observation_id":"88e64ce3-1dea-4d28-8fce-c6cae94592b5","resolution":{"observed_at":"2026-08-11T11:22:57.392673Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.373205Z","title":"Filling conversation ellipsis for better social dialog understanding,","venue":null,"work_id":"adca896a-c110-41b2-adbf-ce287086c795","year":2020},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.763842Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:26658ff422430a2e7a109113e5bb49cf2de8ca5cf69108a6dbe5be0501aa0b47","observation_id":"7d54273a-4860-4fdd-bd69-a2e27b4f9a5a","resolution":{"observed_at":"2026-08-11T11:22:57.377992Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.358784Z","title":"Mitkov,The Oxford handbook of computational linguistics","venue":null,"work_id":"60002194-7f50-4366-abff-0c4b3ec8b3d4","year":2022},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.768335Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:f3cff98a2cedda8cf1930467281f9ce3c79008d5c27509207e8c17ad578256cb","observation_id":"3805f2ee-723b-4c3e-b3d2-89f0d6c20690","resolution":{"observed_at":"2026-08-11T11:22:57.363558Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.342477Z","title":"Anaphora and coreference resolution: A review,","venue":null,"work_id":"701d3fba-9453-4afb-b142-673c9d1e4321","year":2020},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.772812Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:39f6b9992e6b90727ed12ef8b5cb564f1573c13994a9584834f44789578cfed8","observation_id":"00292d6e-3572-46b5-9752-128585074f1d","resolution":{"observed_at":"2026-08-11T11:22:57.348344Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.16130","last_updated":"2025-02-19T10:49:41Z","snapshot_observed_at":"2026-08-16T12:38:40.131901Z","submitted_at":"2024-04-24T18:38:11Z","title":"From Local to Global: A Graph RAG Approach to Query-Focused Summarization","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.16130","snapshot_observed_at":"2026-08-11T11:22:56.777537Z","title":"From local to global: A graph rag approach to query- focused summarization,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.777537Z"},"links":{"cited_paper":"/paper/2404.16130","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:c39a13dd26f2518408fb22dd095b039c8fe17c57a7751448bf64f76aa944214c","observation_id":"7660433f-916d-4a20-8e91-76ec2bf79167","resolution":{"observed_at":"2026-08-11T11:22:56.777537Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.03868","last_updated":"2024-10-02T05:51:53Z","snapshot_observed_at":"2026-08-17T20:25:51.435083Z","submitted_at":"2024-04-05T02:53:51Z","title":"Extract, Define, Canonicalize: An LLM-based Framework for Knowledge Graph Construction","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.03868","snapshot_observed_at":"2026-08-11T11:22:56.783874Z","title":"Extract, define, canonicalize: An llm- based framework for knowledge graph construction,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.783874Z"},"links":{"cited_paper":"/paper/2404.03868","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:980858dcd23e28320c610034c84c01d65201a2ba44d048d89d01868b5a4e5045","observation_id":"3e36c39f-b3fe-41a0-80f8-8d8f09233ac6","resolution":{"observed_at":"2026-08-11T11:22:56.783874Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.10671","last_updated":"2024-09-10T13:25:53Z","snapshot_observed_at":"2026-08-17T11:08:48.802438Z","submitted_at":"2024-07-15T12:35:42Z","title":"Qwen2 Technical Report","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.10671","snapshot_observed_at":"2026-08-11T11:22:56.788561Z","title":"Qwen2 technical report,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.788561Z"},"links":{"cited_paper":"/paper/2407.10671","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:1078c84eea6b844615e79ac59a14aa32ea056a481c19e418117444833686e229","observation_id":"d24dfe24-c8f9-49e2-909c-4909b472331a","resolution":{"observed_at":"2026-08-11T11:22:56.788561Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.327760Z","title":"Mistral-7b-instruct-v0.3 model,","venue":null,"work_id":"62450c21-7faf-4cf6-b8d0-1b649e0a0af4","year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.793739Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:cf08a86bdd21d487dba4b0524291eab040a066a999231f72ba110e265b877f2f","observation_id":"f44c7dc7-1453-4dac-8ab4-83451743b9f2","resolution":{"observed_at":"2026-08-11T11:22:57.332368Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:56.798663Z","title":"Llama 3 model card,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.798663Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:d48ddc6431030b1d419782ea37e8b1be40fc377412f1ef77e7f276fcd1109f2f","observation_id":"27cfddac-bc88-45e5-b4de-b3e79efe66cc","resolution":{"observed_at":"2026-08-11T11:22:56.798663Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:56.803091Z","title":"Team, “Gemma,” 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.803091Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:8de2456cf27cd5eaca15ca3291853a272f0b8967bb3c89fda84f1089c226191e","observation_id":"7a902399-1bbe-4704-9bac-673685245328","resolution":{"observed_at":"2026-08-11T11:22:56.803091Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:56.807530Z","title":"Language mod- els are few-shot learners,","venue":null,"work_id":null,"year":1901},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.807530Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:8a4a94d8f73e37907a33b77f86c251018996a695bdf24c91dd86c3f3cc3aedec","observation_id":"48cab994-c9a8-41c6-875c-f3f5dfed7237","resolution":{"observed_at":"2026-08-11T11:22:56.807530Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1606.05250","last_updated":"2016-10-11T02:42:36Z","snapshot_observed_at":"2026-08-17T14:32:22.468812Z","submitted_at":"2016-06-16T16:36:00Z","title":"SQuAD: 100,000+ Questions for Machine Comprehension of Text","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1606.05250","snapshot_observed_at":"2026-08-11T11:22:56.812475Z","title":"Squad: 100,000+ questions for machine comprehension of text,","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.812475Z"},"links":{"cited_paper":"/paper/1606.05250","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:38015f8d5fad2a0e29e112aeb232351b3cb35924fdb3fac3bf9c11948907f32d","observation_id":"3cfade8e-dcd7-4b77-9909-06065e854648","resolution":{"observed_at":"2026-08-11T11:22:56.812475Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.285019Z","title":"Know what you don’t know: Unanswerable questions for squad,","venue":null,"work_id":"16b97209-fb80-40b8-a671-35a72bf933f1","year":2018},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.818022Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:131d9105deb0c033a59aa68e71de3eba27090f0bbefd44963ccf1f8e95e5f4f6","observation_id":"73425cfc-16c6-4d98-8ede-640a136c86db","resolution":{"observed_at":"2026-08-11T11:22:57.289680Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.270054Z","title":"Mutual: A dataset for multi-turn dialogue reasoning,","venue":null,"work_id":"f9906b45-8c38-47de-9c86-10848560860a","year":2020},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.822950Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:7f613aec763a5b1800a2315fb91f4a8564f1fa582df0f910641b2294398f1fea","observation_id":"a9ee1099-ac90-4ccb-a621-636cee3cc764","resolution":{"observed_at":"2026-08-11T11:22:57.274645Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.02839","last_updated":"2025-05-30T12:01:03Z","snapshot_observed_at":"2026-08-16T14:12:43.451038Z","submitted_at":"2024-03-05T10:20:52Z","title":"An Empirical Study of LLM-as-a-Judge for LLM Evaluation: Fine-tuned Judge Model is not a General Substitute for GPT-4","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.02839","snapshot_observed_at":"2026-08-11T11:22:56.827813Z","title":"An empirical study of llm-as-a-judge for llm evaluation: Fine-tuned judge models are task- specific classifiers,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.827813Z"},"links":{"cited_paper":"/paper/2403.02839","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:105d948aaaf1e7fedd553e2880c4bb05981b9947d4d6f287bf4d699df5838d83","observation_id":"5f8fbf1b-aae6-4303-a4a9-1e1a3577189c","resolution":{"observed_at":"2026-08-11T11:22:56.827813Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.254991Z","title":"Opal: Ontology- aware pretrained language model for end-to-end task-oriented dialogue,","venue":null,"work_id":"9f09dbe5-4bf3-4f4a-bbbd-eb65ddacd22c","year":2023},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.832666Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:38a49d5b87db2efcb2505704ac3f7ef9897b4087ac5c8c0066386ab23e7b21f2","observation_id":"49974e84-fefb-412e-92b3-d155e2db70f1","resolution":{"observed_at":"2026-08-11T11:22:57.259822Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2302.10205","last_updated":"2024-05-27T07:08:31Z","snapshot_observed_at":"2026-08-16T15:54:08.108332Z","submitted_at":"2023-02-20T12:57:12Z","title":"ChatIE: Zero-Shot Information Extraction via Chatting with ChatGPT","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.10205","snapshot_observed_at":"2026-08-11T11:22:56.837496Z","title":"Zero-shot information extraction via chatting with chatgpt,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.837496Z"},"links":{"cited_paper":"/paper/2302.10205","citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:52eb9a4433bf3f51f9be712e9e8f62fd0f6523faece380976ebc88557f910684","observation_id":"02a65745-8f11-4e92-a625-2cef3445f092","resolution":{"observed_at":"2026-08-11T11:22:56.837496Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.238943Z","title":"Testing graph database systems via graph-aware metamorphic relations,","venue":null,"work_id":"63e2c028-7f8a-407f-ab10-34c229f85e25","year":2023},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.842520Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:7f686788d7fed1182413747af99cb6079f3810016ae57e5c418f1a708f7f8e6e","observation_id":"922061ef-e2ae-4efb-b467-dcf04b1a6e54","resolution":{"observed_at":"2026-08-11T11:22:57.244428Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.222938Z","title":"Metamorphic testing for software quality assessment: A study of search engines,","venue":null,"work_id":"b2a3a43f-12ac-432d-be88-161c88ec93d4","year":2015},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.847198Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:3888eaac98e42d852e870b0042daace15b69ed01eb9db9e80e04345ed8cccc0c","observation_id":"e306b403-54b9-42bb-999f-e907642b6b4d","resolution":{"observed_at":"2026-08-11T11:22:57.227902Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.207919Z","title":"Testing and validating machine learning classifiers by metamorphic testing,","venue":null,"work_id":"df8c209c-aabf-4d55-b25a-e66e019531ba","year":2011},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.851876Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:5e9d8a5af7f90c7dc58eff37576c1ee161b5c779c6db4026ed3a09bca064e4ea","observation_id":"ab95c28e-5f95-434d-a4fe-f873152f0a27","resolution":{"observed_at":"2026-08-11T11:22:57.212808Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.193102Z","title":"Mttm: Metamorphic testing for textual content modera- tion software,","venue":null,"work_id":"5591995d-670a-4866-ad43-2885152e8338","year":2023},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.856456Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:eccd69d4883cffa94c08ffd09b02bb718ceb218af66918f4d621e618a2e0f1d4","observation_id":"d3df67be-58e4-4017-98f3-cbb6cbd6d54c","resolution":{"observed_at":"2026-08-11T11:22:57.198165Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.177876Z","title":"Ontology-based metamorphic testing for chatbots,","venue":null,"work_id":"28c1fa9c-ce57-4715-a2a7-6f3e3df8f65f","year":2022},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.861209Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:dcf66594e9e1f3c7888e1efed784c9c847d7fb0d660def42b50aa1985d837ee4","observation_id":"5429a64c-18d3-4640-acf7-84779a841497","resolution":{"observed_at":"2026-08-11T11:22:57.182591Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T11:22:57.162569Z","title":"Knowledge graph driven inference testing for question answering software,","venue":null,"work_id":"cb1d3c47-3fba-4a6c-a939-41f62d7bbb44","year":2024},"citing_paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems","version":4},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-11T11:22:56.865686Z"},"links":{"citing_paper":"/paper/2412.15557"},"observation_digest":"sha256:9c9902e662d2ff272b4b2b4576306cc71964d6a799c0e45dd04c3e2cbd7e93ab","observation_id":"a3389b90-247f-4215-a836-8e2f7d0276a3","resolution":{"observed_at":"2026-08-11T11:22:57.167741Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2412.15557","last_updated":"2026-06-17T03:05:03Z","latest_version":4,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-14T13:47:31.345377Z","submitted_at":"2024-12-20T04:31:03Z","title":"MORTAR: Multi-turn Metamorphic Testing for LLM-based Dialogue Systems"},"reference_resolution":{"displayed":56,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":23,"verified_exact":1,"verified_fuzzy":32},"total_outbound_references":56},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"thesis":"As of 17 August 2026, this Paper Citation Record lists 56 of 56 outbound references and 2 inbound Pith citation observations for arXiv:2412.15557."}