{"as_of":"2026-08-12T13:33:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:41ea97d27ccb6ca4e533e0513e5f08c579310c5db6332b0c3367f02c3a8231d4","coverage":[{"denominator":209,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":100,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T06:00:19.491104Z","state":"measured"},{"denominator":104,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":104,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-12T06:34:41.77262+00:00","state":"measured"},{"denominator":4,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":4,"source":"paper_references, paper_reference_links","source_observed_at":"2026-06-29T12:52:20.911788Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-06-29T12:53:26.388274Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"cited_work":{"arxiv_id":"2506.11102","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.11102","snapshot_observed_at":"2026-06-29T12:53:26.388274Z","title":"Evolutionary perspectives on the evaluation of llm-based ai agents: A comprehensive survey","venue":null,"work_id":"ca103352-c9a6-468c-b222-ed17e7679034","year":2025},"citing_paper":{"arxiv_id":"2507.21046","last_updated":"2026-01-16T20:59:08Z","snapshot_observed_at":"2026-08-01T06:32:44.461162Z","submitted_at":"2025-07-28T17:59:05Z","title":"A Survey of Self-Evolving Agents: What, When, How, and Where to Evolve on the Path to Artificial Super Intelligence","version":4},"reference_index":154,"source":"arxiv_source","source_observed_at":"2026-05-14T22:23:14.621091Z"},"links":{"cited_paper":"/paper/2506.11102","citing_paper":"/paper/2507.21046"},"observation_digest":"sha256:d2b7cd7ec00fdeee5db072daafe8922f414b417fe26c5cf7ee14b506a6c1b872","observation_id":"c18e21d6-4917-4748-9379-93e8566336c1","resolution":{"observed_at":"2026-05-14T22:23:14.879801Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"cited_work":{"arxiv_id":"2506.11102","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.11102","snapshot_observed_at":"2026-06-29T12:53:26.388274Z","title":"Evolutionary perspectives on the evaluation of llm-based ai agents: A comprehensive survey","venue":null,"work_id":"ca103352-c9a6-468c-b222-ed17e7679034","year":2025},"citing_paper":{"arxiv_id":"2604.08224","last_updated":"2026-04-09T13:19:41Z","snapshot_observed_at":"2026-08-02T22:56:00.067549Z","submitted_at":"2026-04-09T13:19:41Z","title":"Externalization in LLM Agents: A Unified Review of Memory, Skills, Protocols and Harness Engineering","version":1},"reference_index":201,"source":"arxiv_source","source_observed_at":"2026-05-10T17:40:14.733882Z"},"links":{"cited_paper":"/paper/2506.11102","citing_paper":"/paper/2604.08224"},"observation_digest":"sha256:aa759a88111c8a620be4f0e6de6167d69cb682790349ab5efe4daec2b655b7cd","observation_id":"3e2c672f-c607-4eae-81a4-f70ce2797404","resolution":{"observed_at":"2026-05-11T06:21:00.461563Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"cited_work":{"arxiv_id":"2506.11102","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.11102","snapshot_observed_at":"2026-06-29T12:53:26.388274Z","title":"Evolutionary perspectives on the evaluation of llm-based ai agents: A comprehensive survey","venue":null,"work_id":"ca103352-c9a6-468c-b222-ed17e7679034","year":2025},"citing_paper":{"arxiv_id":"2605.15710","last_updated":"2026-05-15T08:00:46Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2026-05-15T08:00:46Z","title":"SMMBench: A Benchmark for Source-Distributed Multimodal Agent Memory","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-05-20T19:11:15.761831Z"},"links":{"cited_paper":"/paper/2506.11102","citing_paper":"/paper/2605.15710"},"observation_digest":"sha256:1ce65601cc074b673f01206363bf198f17d12c81132b24adce6211ee51dada1b","observation_id":"0c5748b6-1302-4008-9625-1e82f498a169","resolution":{"observed_at":"2026-05-20T19:13:40.511046Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"cited_work":{"arxiv_id":"2506.11102","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.11102","snapshot_observed_at":"2026-06-29T12:53:26.388274Z","title":"Evolutionary perspectives on the evaluation of llm-based ai agents: A comprehensive survey","venue":null,"work_id":"ca103352-c9a6-468c-b222-ed17e7679034","year":2025},"citing_paper":{"arxiv_id":"2605.28158","last_updated":"2026-05-27T08:41:30Z","snapshot_observed_at":"2026-07-06T23:37:45.223562Z","submitted_at":"2026-05-27T08:41:30Z","title":"OR-Space: A Full-Lifecycle Workspace Benchmark for Industrial Optimization Agents","version":1},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-06-29T12:52:20.911788Z"},"links":{"cited_paper":"/paper/2506.11102","citing_paper":"/paper/2605.28158"},"observation_digest":"sha256:3ba82049cd4ad226532ad4843c7d83ae9f71fa6db2efc3c6590be5354a77c332","observation_id":"068b5531-8028-47f4-a264-5b52127cbf69","resolution":{"observed_at":"2026-06-29T12:53:26.389766Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2506.11102/citation-record","integrity":"/paper/2506.11102/integrity","json":"/paper/2506.11102/citation-record.json","paper":"/paper/2506.11102"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"1706.03762","last_updated":"2023-08-02T00:41:18Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2017-06-12T17:57:34Z","title":"Attention Is All You Need","version":7},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1706.03762","snapshot_observed_at":"2026-08-07T06:00:15.697976Z","title":"Attention is all you need,","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:15.697976Z"},"links":{"cited_paper":"/paper/1706.03762","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:d9dc0c1d44c9698806aaf1b0f694b61d0ff6c4e8d9cdea5f6eeb053d1cb4c0ff","observation_id":"e9f6a353-6bd8-4e4c-ad86-066f7a6b64b0","resolution":{"observed_at":"2026-08-07T06:00:15.697976Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-07T07:30:12.213965Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-07T06:00:15.738449Z","title":"Gpt-4 technical report,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:15.738449Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:a65ed34c7e1403b963c99002c1b5a6ebb35ef8a575534540580e74746b3d243e","observation_id":"20a874fb-0745-42ba-ae6c-4dbe13914100","resolution":{"observed_at":"2026-08-07T06:00:15.738449Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2302.13971","last_updated":"2023-02-27T17:11:15Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-02-27T17:11:15Z","title":"LLaMA: Open and Efficient Foundation Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.13971","snapshot_observed_at":"2026-08-07T06:00:15.823334Z","title":"Llama: Open and efficient foundation language models,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:15.823334Z"},"links":{"cited_paper":"/paper/2302.13971","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:b71ef82d729ee0d8731e4d19d8860234ba93f9c4bf3e6f4ef80637472f7c010d","observation_id":"e4204d53-69d2-4844-9220-e09b75a3b614","resolution":{"observed_at":"2026-08-07T06:00:15.823334Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.11805","last_updated":"2025-05-09T21:04:06Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-12-19T02:39:27Z","title":"Gemini: A Family of Highly Capable Multimodal Models","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.11805","snapshot_observed_at":"2026-08-07T06:00:15.888246Z","title":"Gemini: A family of highly capable multimodal models,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:15.888246Z"},"links":{"cited_paper":"/paper/2312.11805","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:10a3d679607bd1e9d9d9b20faeca59649314899bbcf698aba2d11dd3aea46378","observation_id":"bd209cb1-83af-4bed-a32e-42fe2a9e3eef","resolution":{"observed_at":"2026-08-07T06:00:15.888246Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.16609","last_updated":"2023-09-28T17:07:49Z","snapshot_observed_at":"2026-08-09T21:25:20.369782Z","submitted_at":"2023-09-28T17:07:49Z","title":"Qwen Technical Report","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.16609","snapshot_observed_at":"2026-08-07T06:00:15.977297Z","title":"Qwen technical report,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:15.977297Z"},"links":{"cited_paper":"/paper/2309.16609","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:cbba90ddba375914a23ff1aa9713976631b67da6ba7ce2754983d0a1e4c4fc03","observation_id":"1f1aea1c-d210-439c-826e-ad41f4eb41da","resolution":{"observed_at":"2026-08-07T06:00:15.977297Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.12948","last_updated":"2026-01-04T03:57:36Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-01-22T15:19:35Z","title":"DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.12948","snapshot_observed_at":"2026-08-07T06:00:16.066521Z","title":"Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:16.066521Z"},"links":{"cited_paper":"/paper/2501.12948","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:5a814befd5029633a7dd380c24c5af9d33e5c800d7dfb8d084f138679afa5ee3","observation_id":"9955fec4-7265-4424-801f-20e4dbd9cb3d","resolution":{"observed_at":"2026-08-07T06:00:16.066521Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.13032","last_updated":"2024-07-17T21:44:28Z","snapshot_observed_at":"2026-08-07T16:45:05.155981Z","submitted_at":"2024-07-17T21:44:28Z","title":"Agent-E: From Autonomous Web Navigation to Foundational Design Principles in Agentic Systems","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.13032","snapshot_observed_at":"2026-08-07T06:00:16.156522Z","title":"Agent-e: From autonomous web navigation to foundational design principles in agentic systems,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:16.156522Z"},"links":{"cited_paper":"/paper/2407.13032","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:0331112f6c767a9cf68a48b58cbf995bc24a9b88623e8c853bf80fb12bcd5fc9","observation_id":"69fa2201-e8bc-4705-b486-9f157ea47c3b","resolution":{"observed_at":"2026-08-07T06:00:16.156522Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.16464","last_updated":"2025-06-16T22:23:37Z","snapshot_observed_at":"2026-08-11T21:25:45.556303Z","submitted_at":"2024-10-21T19:46:06Z","title":"Beyond Browsing: API-Based Web Agents","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.16464","snapshot_observed_at":"2026-08-07T06:00:16.253072Z","title":"Beyond browsing: Api-based web agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:16.253072Z"},"links":{"cited_paper":"/paper/2410.16464","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:57b4cbe3e164d97d40c4e97ddb27287d56890a16d9b032f5b4aa0a7bc01f843d","observation_id":"bce1507e-654c-4c19-9d1f-29057183678a","resolution":{"observed_at":"2026-08-07T06:00:16.253072Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.11366","last_updated":"2023-10-10T05:21:45Z","snapshot_observed_at":"2026-08-10T17:44:16.302071Z","submitted_at":"2023-03-20T18:08:50Z","title":"Reflexion: Language Agents with Verbal Reinforcement Learning","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.11366","snapshot_observed_at":"2026-08-07T06:00:16.334469Z","title":"Reflexion: Language agents with verbal reinforcement learning,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:16.334469Z"},"links":{"cited_paper":"/paper/2303.11366","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:64f14baa8a9a5d9d4477e4e54217ecdae6e77bc2bde63b06100a3e67e7eed4c5","observation_id":"b486343d-eeb8-4628-8e1f-34b2dbe3db46","resolution":{"observed_at":"2026-08-07T06:00:16.334469Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.03109","last_updated":"2023-12-29T02:12:03Z","snapshot_observed_at":"2026-08-05T13:54:21.157165Z","submitted_at":"2023-07-06T16:28:35Z","title":"A Survey on Evaluation of Large Language Models","version":9},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.03109","snapshot_observed_at":"2026-08-07T06:00:16.424040Z","title":"A survey on evaluation of large language models,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:16.424040Z"},"links":{"cited_paper":"/paper/2307.03109","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:cb72187970ca9fdfcb3e9af35d1f3b5af62245962b82b7a593e23fb7c29a5d30","observation_id":"25195080-ca4c-4c89-b675-24aa4716ba9c","resolution":{"observed_at":"2026-08-07T06:00:16.424040Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.04069","last_updated":"2024-10-03T13:51:53Z","snapshot_observed_at":"2026-08-09T00:13:53.804763Z","submitted_at":"2024-07-04T17:15:37Z","title":"A Systematic Survey and Critical Review on Evaluating Large Language Models: Challenges, Limitations, and Recommendations","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.04069","snapshot_observed_at":"2026-08-07T06:00:16.519703Z","title":"A systematic survey and critical review on evaluating large language models: Challenges, limitations, and recommendations,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:16.519703Z"},"links":{"cited_paper":"/paper/2407.04069","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:ac01dbbd6594794c50dacaad336f860b289f84c11a06eee6853d0ae8b29c6968","observation_id":"67aa1f7b-6ad2-4d05-81d8-0bebbd80e45a","resolution":{"observed_at":"2026-08-07T06:00:16.519703Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.19736","last_updated":"2023-11-25T17:35:12Z","snapshot_observed_at":"2026-07-06T16:40:35.590974Z","submitted_at":"2023-10-30T17:00:52Z","title":"Evaluating Large Language Models: A Comprehensive Survey","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.19736","snapshot_observed_at":"2026-08-07T06:00:16.603597Z","title":"Evaluating large language models: A comprehensive survey,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:16.603597Z"},"links":{"cited_paper":"/paper/2310.19736","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:8b20b0cd20b0db2917186434b6ce75ae9e354c5038800b5d9b64a3c2b901b093","observation_id":"a6a852ba-7372-4622-9b0c-2306905712da","resolution":{"observed_at":"2026-08-07T06:00:16.603597Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T11:58:24.516369Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-07T06:00:16.716446Z","title":"Evaluating large lan- guage models trained on code,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:16.716446Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:0efa5f30e589a6e9801a172ec1914867e464fa79bae0ca44dec3d261ec015fd6","observation_id":"e22630de-a20b-4ab0-a2ba-22e7415e6c38","resolution":{"observed_at":"2026-08-07T06:00:16.716446Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.01861","last_updated":"2023-08-14T09:07:00Z","snapshot_observed_at":"2026-08-03T22:16:12.497107Z","submitted_at":"2023-08-03T16:31:02Z","title":"ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.01861","snapshot_observed_at":"2026-08-07T06:00:16.785966Z","title":"Classeval: A manually-crafted benchmark for evaluating llms on class-level code generation,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:16.785966Z"},"links":{"cited_paper":"/paper/2308.01861","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:8c1f6007126d494a8f4522e0f0e58eb2bb0db9dd0daa6e4a8e24fb30b26b0c31","observation_id":"6c35e0c2-bf2a-4915-8d74-55c1a2c5b3a0","resolution":{"observed_at":"2026-08-07T06:00:16.785966Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:16.847762Z","title":"Codereval: A benchmark of pragmatic code generation with generative pre-trained models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:16.847762Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:08d105c2c1ce892659ff3708e1f1dc330d2cbd42b14bfad4a56a3590079abf76","observation_id":"8725aa85-b4ef-4110-94c3-4875641fb7c4","resolution":{"observed_at":"2026-08-07T06:00:16.847762Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.01240","last_updated":"2023-08-02T15:54:22Z","snapshot_observed_at":"2026-07-06T16:01:45.794350Z","submitted_at":"2023-08-02T15:54:22Z","title":"Evaluating Instruction-Tuned Large Language Models on Code Comprehension and Generation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.01240","snapshot_observed_at":"2026-08-07T06:00:16.923044Z","title":"Evaluating instruction-tuned large language models on code comprehension and generation,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:16.923044Z"},"links":{"cited_paper":"/paper/2308.01240","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:b1fb1bdc64d4c8e9e6ac737fed2d11287e11908fb5d0fbbb6264593834a546d9","observation_id":"a9a44159-a09f-4e11-8a89-a47582ee1cb6","resolution":{"observed_at":"2026-08-07T06:00:16.923044Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.16614","last_updated":"2025-02-23T15:36:43Z","snapshot_observed_at":"2026-08-10T20:16:41.516189Z","submitted_at":"2025-02-23T15:36:43Z","title":"CodeCriticBench: A Holistic Code Critique Benchmark for Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.16614","snapshot_observed_at":"2026-08-07T06:00:17.038881Z","title":"Codecriticbench: A holistic code critique benchmark for large language models,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:17.038881Z"},"links":{"cited_paper":"/paper/2502.16614","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:025aa034b5a67a86366754256b853d52b1ead5816c89ac44de482ad1385f7cb7","observation_id":"bb73fe5c-1553-4d57-80b8-eec45055522d","resolution":{"observed_at":"2026-08-07T06:00:17.038881Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.03065","last_updated":"2024-01-05T20:53:51Z","snapshot_observed_at":"2026-07-06T17:12:10.787061Z","submitted_at":"2024-01-05T20:53:51Z","title":"CRUXEval: A Benchmark for Code Reasoning, Understanding and Execution","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.03065","snapshot_observed_at":"2026-08-07T06:00:17.129393Z","title":"Cruxeval: A benchmark for code reasoning, understanding and execution,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:17.129393Z"},"links":{"cited_paper":"/paper/2401.03065","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:0e4e5eede55c1e5a24338f2996b2fc6ac3b81b3897da01d6bd1c81eea0bc14e2","observation_id":"6b00341c-253f-4bcc-9dbe-67c6ddb7b03f","resolution":{"observed_at":"2026-08-07T06:00:17.129393Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.15877","last_updated":"2025-04-01T08:36:44Z","snapshot_observed_at":"2026-07-31T19:00:59.311189Z","submitted_at":"2024-06-22T15:52:04Z","title":"BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.15877","snapshot_observed_at":"2026-08-07T06:00:17.220729Z","title":"Bigcodebench: Bench- marking code generation with diverse function calls and complex instructions,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:17.220729Z"},"links":{"cited_paper":"/paper/2406.15877","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:9c9bdec3b34da7f7b54ffddbba3d9dcdadaf9c259d181a20b541c22c91187613","observation_id":"4f4399a5-4ada-423b-b1de-9ac0369b8db8","resolution":{"observed_at":"2026-08-07T06:00:17.220729Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2212.09248","last_updated":"2022-12-19T05:06:00Z","snapshot_observed_at":"2026-08-10T22:00:21.920551Z","submitted_at":"2022-12-19T05:06:00Z","title":"Natural Language to Code Generation in Interactive Data Science Notebooks","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2212.09248","snapshot_observed_at":"2026-08-07T06:00:17.290883Z","title":"Natural language to code generation in interactive data science notebooks,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:17.290883Z"},"links":{"cited_paper":"/paper/2212.09248","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:79b35de77d08714f4b115e5dffa58390b1866a09374039fca0d22dab6490e993","observation_id":"fb373681-2cd1-4688-80b1-e656e8507ee3","resolution":{"observed_at":"2026-08-07T06:00:17.290883Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.06770","last_updated":"2024-11-11T23:05:04Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-10-10T16:47:29Z","title":"SWE-bench: Can Language Models Resolve Real-World GitHub Issues?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.06770","snapshot_observed_at":"2026-08-07T06:00:17.379245Z","title":"Swe-bench: Can language models resolve real-world github issues?","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:17.379245Z"},"links":{"cited_paper":"/paper/2310.06770","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:925747ceffad253d651ecf19dc469941eef915368adf5b655a73121e9018da45","observation_id":"38c40404-8e4a-442d-92f3-7c537a4581bd","resolution":{"observed_at":"2026-08-07T06:00:17.379245Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.03859","last_updated":"2024-10-04T18:48:58Z","snapshot_observed_at":"2026-07-06T19:28:08.374354Z","submitted_at":"2024-10-04T18:48:58Z","title":"SWE-bench Multimodal: Do AI Systems Generalize to Visual Software Domains?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.03859","snapshot_observed_at":"2026-08-07T06:00:17.468272Z","title":"Swe-bench multimodal: Do ai systems generalize to visual software domains?","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:17.468272Z"},"links":{"cited_paper":"/paper/2410.03859","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:58001c05a2616106d9cb4f38462b0296c4266adb1b18803c77f4fa3f23d12ad6","observation_id":"9a21d675-1c13-4335-81f9-84ac9f795873","resolution":{"observed_at":"2026-08-07T06:00:17.468272Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.20285","last_updated":"2025-04-02T04:13:19Z","snapshot_observed_at":"2026-07-06T19:40:12.973874Z","submitted_at":"2024-10-26T22:45:56Z","title":"SWE-Search: Enhancing Software Agents with Monte Carlo Tree Search and Iterative Refinement","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.20285","snapshot_observed_at":"2026-08-07T06:00:17.591067Z","title":"Swe- search: Enhancing software agents with monte carlo tree search and iterative refinement,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:17.591067Z"},"links":{"cited_paper":"/paper/2410.20285","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:a86b29dfa4213d670c9d0eff83911774d97d54b8448ffbbb70757c16b44b2109","observation_id":"a1309315-93d3-4e28-9d01-2ab65f09bbd6","resolution":{"observed_at":"2026-08-07T06:00:17.591067Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.14354","last_updated":"2024-08-26T15:30:05Z","snapshot_observed_at":"2026-08-10T18:57:19.903703Z","submitted_at":"2024-08-26T15:30:05Z","title":"SWE-bench-java: A GitHub Issue Resolving Benchmark for Java","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.14354","snapshot_observed_at":"2026-08-07T06:00:17.668524Z","title":"Swe-bench-java: A github issue resolving benchmark for java, 2024,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:17.668524Z"},"links":{"cited_paper":"/paper/2408.14354","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:bf97009dc22779db1c9b7b39ce737e042a55a374797a68c706b3733b15fe5bc0","observation_id":"a6f4fa44-f487-4298-89f8-fc866145363d","resolution":{"observed_at":"2026-08-07T06:00:17.668524Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:17.755574Z","title":"Swe arena: An open evaluation platform for automated software engineering,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:17.755574Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:0d3b749bd8f86c56a61a694913408a558ccd335aeef84a0037cc80ebd303f07c","observation_id":"ac6f0cbc-2bd6-4dc9-8551-cdf4d36fc239","resolution":{"observed_at":"2026-08-07T06:00:17.755574Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.16655","last_updated":"2025-01-28T02:38:56Z","snapshot_observed_at":"2026-08-10T18:57:26.796740Z","submitted_at":"2025-01-28T02:38:56Z","title":"Large Language Model Critics for Execution-Free Evaluation of Code Changes","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.16655","snapshot_observed_at":"2026-08-07T06:00:17.825915Z","title":"Large language model critics for execution-free evaluation of code changes,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:17.825915Z"},"links":{"cited_paper":"/paper/2501.16655","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:74bf240f6f2dd966554f9a5cdfa26005c612da5c4a6d2a5ceffab0e11a829716","observation_id":"260f0122-a06f-458b-8013-6a447543a989","resolution":{"observed_at":"2026-08-07T06:00:17.825915Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.06992","last_updated":"2024-10-10T13:13:09Z","snapshot_observed_at":"2026-08-08T05:32:33.029750Z","submitted_at":"2024-10-09T15:38:53Z","title":"SWE-Bench+: Enhanced Coding Benchmark for LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.06992","snapshot_observed_at":"2026-08-07T06:00:17.909889Z","title":"Swe-bench+: En- hanced coding benchmark for llms,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:17.909889Z"},"links":{"cited_paper":"/paper/2410.06992","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:04ac7078f8ea497b4ffe06cbc8f79b65fd2b8244d658ebd97bacac66001041af","observation_id":"813acb66-9098-400d-ad9c-6e5b2a189a9b","resolution":{"observed_at":"2026-08-07T06:00:17.909889Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.03091","last_updated":"2023-10-04T01:13:49Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-06-05T17:59:41Z","title":"RepoBench: Benchmarking Repository-Level Code Auto-Completion Systems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.03091","snapshot_observed_at":"2026-08-07T06:00:17.987699Z","title":"Repobench: Bench- marking repository-level code auto-completion systems,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:17.987699Z"},"links":{"cited_paper":"/paper/2306.03091","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:1b75092e6660a2b46cdef1e8651f33c7f9d13743e783707fc71195987ed9c8b9","observation_id":"e418d1de-4524-4de4-8d16-e35ddd9d35d4","resolution":{"observed_at":"2026-08-07T06:00:17.987699Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:18.114116Z","title":"Swt- bench: Testing and validating real-world bug-fixes with JOURNAL OF LATEX CLASS FILES, VOL. 14, NO. 8, AUGUST 2015 15 code agents,","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:18.114116Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:44ae8c09598ae7bd721a369b363de6d85b86d744307db0c2ca364ec8a7a24590","observation_id":"f139a15e-d5af-4b93-a480-42d0a4391586","resolution":{"observed_at":"2026-08-07T06:00:18.114116Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.07924","last_updated":"2024-06-05T13:23:49Z","snapshot_observed_at":"2026-07-06T15:54:28.831510Z","submitted_at":"2023-07-16T02:11:34Z","title":"ChatDev: Communicative Agents for Software Development","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.07924","snapshot_observed_at":"2026-08-07T06:00:18.220143Z","title":"Chatdev: Communicative agents for software development,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:18.220143Z"},"links":{"cited_paper":"/paper/2307.07924","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:e17813782d678a43ed5f5b931dcbaa43413ce9f3c3de7f48feb85504162c1d66","observation_id":"c9b9dfe3-3ea2-422c-ac78-ec6c47107748","resolution":{"observed_at":"2026-08-07T06:00:18.220143Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.08604","last_updated":"2024-12-14T09:45:51Z","snapshot_observed_at":"2026-08-06T13:32:36.381010Z","submitted_at":"2024-03-13T15:13:44Z","title":"Prompting Large Language Models to Tackle the Full Software Development Lifecycle: A Case Study","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.08604","snapshot_observed_at":"2026-08-07T06:00:18.380320Z","title":"Prompting large language models to tackle the full software development lifecycle: A case study,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:18.380320Z"},"links":{"cited_paper":"/paper/2403.08604","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:d37c8f599f83bf499698b3f91c214b096b05949e2ed603803440a640b282d061","observation_id":"8794d234-6f6e-4577-af53-7fe360f293b7","resolution":{"observed_at":"2026-08-07T06:00:18.380320Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.09835","last_updated":"2024-08-21T13:36:30Z","snapshot_observed_at":"2026-07-06T16:48:33.164024Z","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.09835","snapshot_observed_at":"2026-08-07T06:00:18.492754Z","title":"Ml-bench: Evaluating large lan- guage models and agents for machine learning tasks on repository-level code,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:18.492754Z"},"links":{"cited_paper":"/paper/2311.09835","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:e6cf57671364695b81e061b5c536648c74ac4825896beaa5bdeb27bf13297a3f","observation_id":"d59ecd38-95a2-4267-810e-79a9d494d2f9","resolution":{"observed_at":"2026-08-07T06:00:18.492754Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.16732","last_updated":"2024-08-03T03:00:43Z","snapshot_observed_at":"2026-07-06T18:50:52.800619Z","submitted_at":"2024-07-23T15:23:14Z","title":"PyBench: Evaluating LLM Agent on various real-world coding tasks","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.16732","snapshot_observed_at":"2026-08-07T06:00:18.652929Z","title":"Pybench: Evaluating llm agent on various real-world coding tasks,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:18.652929Z"},"links":{"cited_paper":"/paper/2407.16732","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:3d2e7d01b160889604b0d3fe2cab19ed0a6ec83ce220a64be3ffbf0bc8607bab","observation_id":"64669182-ef2a-4c15-8438-b5fbcd23cb2e","resolution":{"observed_at":"2026-08-07T06:00:18.652929Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:18.819234Z","title":"World of bits: An open- domain platform for web-based agents,","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:18.819234Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:671896aa47ab690d093f19b1eed19820a844bdfa1097e55f4e9102da34af53c8","observation_id":"cfa6a7b2-935b-4330-954d-b906b47d2b12","resolution":{"observed_at":"2026-08-07T06:00:18.819234Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1802.08802","last_updated":"2018-02-24T05:32:47Z","snapshot_observed_at":"2026-07-06T06:25:09.401059Z","submitted_at":"2018-02-24T05:32:47Z","title":"Reinforcement Learning on Web Interfaces Using Workflow-Guided Exploration","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1802.08802","snapshot_observed_at":"2026-08-07T06:00:18.979667Z","title":"Reinforcement learning on web interfaces using workflow-guided exploration,","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:18.979667Z"},"links":{"cited_paper":"/paper/1802.08802","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:c613d7fd518ceda71b2c36755631b18594f280f3001bb641c2b93b6afc336da0","observation_id":"d6a0b09c-4fd8-442b-9c61-e7b4509ca6bb","resolution":{"observed_at":"2026-08-07T06:00:18.979667Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2207.01206","last_updated":"2023-02-08T01:39:30Z","snapshot_observed_at":"2026-07-06T13:27:22.300465Z","submitted_at":"2022-07-04T05:30:22Z","title":"WebShop: Towards Scalable Real-World Web Interaction with Grounded Language Agents","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2207.01206","snapshot_observed_at":"2026-08-07T06:00:19.147928Z","title":"Webshop: Towards scalable real-world web interaction with grounded language agents,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.147928Z"},"links":{"cited_paper":"/paper/2207.01206","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:0064b91f85e25a204dc1c6a04ec9b0ea0b6377bb972f7db9f561cd4003d16e72","observation_id":"28696c09-9fb9-4fee-a302-0c0c984f2934","resolution":{"observed_at":"2026-08-07T06:00:19.147928Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.06070","last_updated":"2023-12-09T05:57:46Z","snapshot_observed_at":"2026-07-06T15:40:50.807376Z","submitted_at":"2023-06-09T17:44:31Z","title":"Mind2Web: Towards a Generalist Agent for the Web","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.06070","snapshot_observed_at":"2026-08-07T06:00:19.200192Z","title":"Mind2web: Towards a generalist agent for the web,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.200192Z"},"links":{"cited_paper":"/paper/2306.06070","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:1f73714a250507b271b2c08284e303f0059e9578ce970432f7cf3c9b72edb658","observation_id":"a000fe7e-1b6c-4350-857e-19800d4bce97","resolution":{"observed_at":"2026-08-07T06:00:19.200192Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.13919","last_updated":"2024-06-06T18:37:34Z","snapshot_observed_at":"2026-08-09T21:47:22.232349Z","submitted_at":"2024-01-25T03:33:18Z","title":"WebVoyager: Building an End-to-End Web Agent with Large Multimodal Models","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.13919","snapshot_observed_at":"2026-08-07T06:00:19.208743Z","title":"Webvoyager: Building an end- to-end web agent with large multimodal models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.208743Z"},"links":{"cited_paper":"/paper/2401.13919","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:0c2d758ae5b270fcac14e8206fee1a30967f781d9cc58a2b2f0f0b946aafe743","observation_id":"32195e48-97c1-47c4-91bc-c333b7e5b9a8","resolution":{"observed_at":"2026-08-07T06:00:19.208743Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.213819Z","title":"Weblinx: Real-world website navigation with multi-turn dialogue,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.213819Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:085e7603e5125c1cbd944defa05ea64dcbc5e5c9fa4b48310c673f93befd9f82","observation_id":"16c6613a-41f3-443b-9ed3-312f57df1290","resolution":{"observed_at":"2026-08-07T06:00:19.213819Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.13854","last_updated":"2024-04-16T15:13:18Z","snapshot_observed_at":"2026-08-06T12:47:01.809185Z","submitted_at":"2023-07-25T22:59:32Z","title":"WebArena: A Realistic Web Environment for Building Autonomous Agents","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.13854","snapshot_observed_at":"2026-08-07T06:00:19.217913Z","title":"Webarena: A realistic web environment for building autonomous agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.217913Z"},"links":{"cited_paper":"/paper/2307.13854","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:459c4be7cb91358bb6fed22cb773f70487a9cc83c162a319c9ac411f7fe19f23","observation_id":"b841e096-a322-4e05-b1e9-be5b2f421b13","resolution":{"observed_at":"2026-08-07T06:00:19.217913Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.13649","last_updated":"2024-06-06T02:01:09Z","snapshot_observed_at":"2026-08-05T16:01:54.847394Z","submitted_at":"2024-01-24T18:35:21Z","title":"VisualWebArena: Evaluating Multimodal Agents on Realistic Visual Web Tasks","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.13649","snapshot_observed_at":"2026-08-07T06:00:19.222011Z","title":"Visualwebarena: Evaluating multimodal agents on realistic visual web tasks,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.222011Z"},"links":{"cited_paper":"/paper/2401.13649","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:f94201d0ed96602307d118463a853049afa160accfc5361e3bd0c307663c5fbe","observation_id":"2ab2c25f-bea8-497c-9224-96dacde30869","resolution":{"observed_at":"2026-08-07T06:00:19.222011Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.07718","last_updated":"2024-07-23T06:19:28Z","snapshot_observed_at":"2026-08-02T12:09:24.340284Z","submitted_at":"2024-03-12T14:58:45Z","title":"WorkArena: How Capable Are Web Agents at Solving Common Knowledge Work Tasks?","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.07718","snapshot_observed_at":"2026-08-07T06:00:19.226003Z","title":"Workarena: How capable are web agents at solving common knowledge work tasks?","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.226003Z"},"links":{"cited_paper":"/paper/2403.07718","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:14b94ddc66611f7c8e579be9999e90068ddcc21430504d4c94dbfe4492d76212","observation_id":"d16271a0-9a49-4f04-8f55-39f65d233eca","resolution":{"observed_at":"2026-08-07T06:00:19.226003Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.05291","last_updated":"2025-02-05T21:50:07Z","snapshot_observed_at":"2026-08-11T11:37:13.973526Z","submitted_at":"2024-07-07T07:15:49Z","title":"WorkArena++: Towards Compositional Planning and Reasoning-based Common Knowledge Work Tasks","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.05291","snapshot_observed_at":"2026-08-07T06:00:19.230822Z","title":"Workarena++: Towards compositional planning and reasoning-based common knowledge work tasks,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.230822Z"},"links":{"cited_paper":"/paper/2407.05291","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:7b4675d540e7ff11153b60818b7fc0eba758b6f1f1b6cc64aaeffdb6186a637a","observation_id":"630117aa-28ce-4993-80b4-114cec9d210a","resolution":{"observed_at":"2026-08-07T06:00:19.230822Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.235463Z","title":"Mmina: Benchmarking multihop multimodal internet agents,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.235463Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:dbcc69611d15feb1ec76d401ee2605f82eabc48b62794a799804bd85e23fe82e","observation_id":"1186c679-70d6-4b64-88d5-97feafe877be","resolution":{"observed_at":"2026-08-07T06:00:19.235463Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.15711","last_updated":"2024-10-21T15:45:31Z","snapshot_observed_at":"2026-08-05T17:32:28.104029Z","submitted_at":"2024-07-22T15:18:45Z","title":"AssistantBench: Can Web Agents Solve Realistic and Time-Consuming Tasks?","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.15711","snapshot_observed_at":"2026-08-07T06:00:19.244078Z","title":"Assistantbench: Can web agents solve realistic and time-consuming tasks?","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.244078Z"},"links":{"cited_paper":"/paper/2407.15711","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:113ffd5124a0fd32574d397c67420e57e5d8a288abf8d59839d68cbbe1a9e294","observation_id":"f46d40d4-b103-481d-bb00-172bd36a716b","resolution":{"observed_at":"2026-08-07T06:00:19.244078Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.12373","last_updated":"2024-07-16T06:19:21Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-06-18T07:58:33Z","title":"WebCanvas: Benchmarking Web Agents in Online Environments","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.12373","snapshot_observed_at":"2026-08-07T06:00:19.248544Z","title":"Webcanvas: Benchmarking web agents in online environments,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.248544Z"},"links":{"cited_paper":"/paper/2406.12373","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:6a03b64655946a8544a2083178bbb64bc593787cc36d2a82a7d42a08f904a25c","observation_id":"b1cf617c-6338-4d35-ba44-47b9195a9237","resolution":{"observed_at":"2026-08-07T06:00:19.248544Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.06703","last_updated":"2026-06-04T09:59:28Z","snapshot_observed_at":"2026-08-05T07:12:18.604381Z","submitted_at":"2024-10-09T09:13:38Z","title":"ST-WebAgentBench: A Benchmark for Evaluating Safety and Trustworthiness in Web Agents","version":7},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.06703","snapshot_observed_at":"2026-08-07T06:00:19.252726Z","title":"St-webagentbench: A benchmark for evaluating safety and trustworthiness in web agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.252726Z"},"links":{"cited_paper":"/paper/2410.06703","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:d81bbfc89f652c382b29a2b57c06d45fbf9f7da3ed156d2cfeaa6420d47ca989","observation_id":"4662cf8d-cc72-4bb3-ad01-cc6b06f34723","resolution":{"observed_at":"2026-08-07T06:00:19.252726Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.19100","last_updated":"2025-02-15T05:19:38Z","snapshot_observed_at":"2026-07-06T19:39:19.409512Z","submitted_at":"2024-10-24T19:03:01Z","title":"VideoWebArena: Evaluating Long Context Multimodal Agents with Video Understanding Web Tasks","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.19100","snapshot_observed_at":"2026-08-07T06:00:19.256618Z","title":"Videowebarena: Evaluating long context multimodal agents with video understanding web tasks,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.256618Z"},"links":{"cited_paper":"/paper/2410.19100","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:1f05098c66f921c65c332774949fe4e1b322bb36a0768866cfdf0dcb54d46f8b","observation_id":"9c94bc8b-9eed-45ac-aa11-f94971c1673d","resolution":{"observed_at":"2026-08-07T06:00:19.256618Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.11905","last_updated":"2025-02-22T01:33:54Z","snapshot_observed_at":"2026-07-06T17:46:25.142387Z","submitted_at":"2024-03-18T16:06:30Z","title":"Tur[k]ingBench: A Challenge Benchmark for Web Agents","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.11905","snapshot_observed_at":"2026-08-07T06:00:19.260739Z","title":"Tur[k]ingbench: A challenge benchmark for web agents,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.260739Z"},"links":{"cited_paper":"/paper/2403.11905","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:f690e9d181ae530575952a42e187dd99266089be19e75c4fd52d381fa8b6cf10","observation_id":"2078033d-11f9-4c4b-aa9a-50f46a02f291","resolution":{"observed_at":"2026-08-07T06:00:19.260739Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.07919","last_updated":"2025-07-24T17:45:05Z","snapshot_observed_at":"2026-08-07T17:14:45.577222Z","submitted_at":"2025-03-10T23:50:30Z","title":"BEARCUBS: A benchmark for computer-using web agents","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.07919","snapshot_observed_at":"2026-08-07T06:00:19.265264Z","title":"Bearcubs: A benchmark for computer-using web agents,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.265264Z"},"links":{"cited_paper":"/paper/2503.07919","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:0d8cee8eb0a2be82cd5bfb3f6030e0ec492a0f113d7089ff4d26a15729fb3f72","observation_id":"f07bc4b2-1d36-420b-9040-ebb798af31bc","resolution":{"observed_at":"2026-08-07T06:00:19.265264Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.14161","last_updated":"2025-09-10T08:35:19Z","snapshot_observed_at":"2026-08-01T16:27:28.241667Z","submitted_at":"2024-12-18T18:55:40Z","title":"TheAgentCompany: Benchmarking LLM Agents on Consequential Real World Tasks","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.14161","snapshot_observed_at":"2026-08-07T06:00:19.269330Z","title":"Theagentcompany: Benchmarking llm agents on consequential real world tasks,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.269330Z"},"links":{"cited_paper":"/paper/2412.14161","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:a19852a3e74ab9c0ed11fb0f94f337bf7f07f3617be09bd4ff1dd3aead3c42ef","observation_id":"5e668517-a3fd-4001-90bb-2b2b25d90438","resolution":{"observed_at":"2026-08-07T06:00:19.269330Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.273386Z","title":"Waber: Evaluating reliability and efficiency of web agents with existing benchmarks,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.273386Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:d07f380bb45940c94b9063eccd710d68a0bc8668775b4176a696f2bbecae7c78","observation_id":"2d4cc33b-2ebb-423f-aa49-f9c9382bc14e","resolution":{"observed_at":"2026-08-07T06:00:19.273386Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.277558Z","title":"Visualagentbench: Towards large multimodal models as visual foundation agents,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.277558Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:0a83508fe3616e7b765ce4f2a541dea6560de53021685e7f2ff3c51259e277ef","observation_id":"4b2137e9-47f1-41d3-ba0c-d1117c0951a1","resolution":{"observed_at":"2026-08-07T06:00:19.277558Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.286579Z","title":"An illusion of progress? assessing the current state of web agents,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.286579Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:bd748604d3bc030c0a65490acc9b0d2270a1f8ff25315c116f85680b57645802","observation_id":"462df6dc-603f-4a8e-bcde-f53f8d7eae9f","resolution":{"observed_at":"2026-08-07T06:00:19.286579Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.282162Z","title":"Available: https://arxiv.org/abs/2408","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.282162Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:1ab3a881707b3238044417af12c302be3aa81bddfde74b00849fdb2a120113a0","observation_id":"3f6c04dd-390b-4807-87bb-94cf5d86252d","resolution":{"observed_at":"2026-08-07T06:00:19.282162Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.17553","last_updated":"2024-07-21T23:16:13Z","snapshot_observed_at":"2026-07-06T17:36:16.016176Z","submitted_at":"2024-02-27T14:47:53Z","title":"OmniACT: A Dataset and Benchmark for Enabling Multimodal Generalist Autonomous Agents for Desktop and Web","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.17553","snapshot_observed_at":"2026-08-07T06:00:19.295587Z","title":"Omniact: A dataset and benchmark for enabling multimodal generalist autonomous agents for desktop and web,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.295587Z"},"links":{"cited_paper":"/paper/2402.17553","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:1d43466be74e251b2f8c3bff02602a4081de1700576b887fccc9b9480af949ca","observation_id":"3a93fdb7-817a-49d1-b70e-8ece429887cd","resolution":{"observed_at":"2026-08-07T06:00:19.295587Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2504.11543","last_updated":"2025-04-17T16:28:46Z","snapshot_observed_at":"2026-08-08T08:50:08.960564Z","submitted_at":"2025-04-15T18:22:55Z","title":"REAL: Benchmarking Autonomous Agents on Deterministic Simulations of Real Websites","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2504.11543","snapshot_observed_at":"2026-08-07T06:00:19.291115Z","title":"Real: Benchmarking autonomous agents on deterministic simulations of real websites,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.291115Z"},"links":{"cited_paper":"/paper/2504.11543","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:870023bc3b0e926f6d6eea8d01f10dbcd8e38e2966b2417e7d070bdd583406c9","observation_id":"ef6a25c2-4649-443a-b98a-a168c8efa59a","resolution":{"observed_at":"2026-08-07T06:00:19.291115Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.308312Z","title":"Osworld: Benchmarking multimodal agents for open-ended tasks in real com- puter environments,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.308312Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:5a6b9cbca4995eea355540ab9cc4aa7f2ebc3e39ad31d31da7f7298de7d08f6d","observation_id":"3bac919f-dc2c-4aae-ba7e-4e4107315075","resolution":{"observed_at":"2026-08-07T06:00:19.308312Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.300016Z","title":"Chatshop: Interactive information seeking with language agents,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.300016Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:3fea86f39ea3149bdb03f9d0ba3b63c5ef11221f9677df223ab05123d206a0c3","observation_id":"7f0563c7-aef4-4f8c-a779-69619ed51102","resolution":{"observed_at":"2026-08-07T06:00:19.300016Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.303915Z","title":"Available: https://arxiv.org/abs/2404","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.303915Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:b194ae2f7bacf72620af4cb6b8c5818d812a04ee569787337385c2f265ecb247","observation_id":"dda98239-a6b9-4277-95b6-c42b36d3fad9","resolution":{"observed_at":"2026-08-07T06:00:19.303915Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.320520Z","title":"Omniact: A dataset and benchmark for enabling multimodal generalist JOURNAL OF LATEX CLASS FILES, VOL. 14, NO. 8, AUGUST 2015 16 autonomous agents for desktop and web,","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.320520Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:a76ae355123eb76480ff5421a91ba8ef2b15979b84ff1aae27075c9d969abfb4","observation_id":"b19e26d6-6b73-4def-8ebd-53bbd598a4f0","resolution":{"observed_at":"2026-08-07T06:00:19.320520Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.08264","last_updated":"2024-09-13T20:17:13Z","snapshot_observed_at":"2026-08-05T14:11:56.506073Z","submitted_at":"2024-09-12T17:56:43Z","title":"Windows Agent Arena: Evaluating Multi-Modal OS Agents at Scale","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.08264","snapshot_observed_at":"2026-08-07T06:00:19.312139Z","title":"Windows agent arena: Eval- uating multi-modal os agents at scale,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.312139Z"},"links":{"cited_paper":"/paper/2409.08264","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:5a2a1c37e9e32ba0cf4b0e2e43e69e7c147c19ca3547b1214607b0cf6fac05b7","observation_id":"a74440e1-7362-4698-933a-030d5e0788dd","resolution":{"observed_at":"2026-08-07T06:00:19.312139Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.17918","last_updated":"2025-02-14T08:13:39Z","snapshot_observed_at":"2026-07-06T17:51:28.420385Z","submitted_at":"2024-03-26T17:54:15Z","title":"AgentStudio: A Toolkit for Building General Virtual Agents","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.17918","snapshot_observed_at":"2026-08-07T06:00:19.316341Z","title":"Agentstudio: A toolkit for building general virtual agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.316341Z"},"links":{"cited_paper":"/paper/2403.17918","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:74e04606788e111138e99587cebf6d1f24a2ba6d7703c737ecaffe42ef16f519","observation_id":"926aa596-72e1-4422-8377-9852b44ba4a2","resolution":{"observed_at":"2026-08-07T06:00:19.316341Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2005.03776","last_updated":"2020-06-05T02:11:56Z","snapshot_observed_at":"2026-08-05T14:55:40.170766Z","submitted_at":"2020-05-07T21:41:40Z","title":"Mapping Natural Language Instructions to Mobile UI Action Sequences","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2005.03776","snapshot_observed_at":"2026-08-07T06:00:19.333127Z","title":"Mapping natural language instruc- tions to mobile ui action sequences,","venue":null,"work_id":null,"year":2005},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":64,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.333127Z"},"links":{"cited_paper":"/paper/2005.03776","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:2976bc1245f479d682b3b368f074ff8d85c9e9740d273e74ec72c8d6ece5cde2","observation_id":"1161f127-6a82-41d7-a01d-7c7dc859b437","resolution":{"observed_at":"2026-08-07T06:00:19.333127Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.19056","last_updated":"2024-07-26T19:27:17Z","snapshot_observed_at":"2026-08-09T13:28:13.876784Z","submitted_at":"2024-07-26T19:27:17Z","title":"OfficeBench: Benchmarking Language Agents across Multiple Applications for Office Automation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.19056","snapshot_observed_at":"2026-08-07T06:00:19.324265Z","title":"Officebench: Benchmarking language agents across multiple applications for office automation,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":65,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.324265Z"},"links":{"cited_paper":"/paper/2407.19056","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:bc1e9df1fe3453dd6e68286a717764de9bb39b8cc6b35e91d5debca85c9c04d8","observation_id":"7c9775cf-fff2-41b9-8e65-cec65a483072","resolution":{"observed_at":"2026-08-07T06:00:19.324265Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.14282","last_updated":"2025-02-21T02:54:09Z","snapshot_observed_at":"2026-08-10T04:59:23.156871Z","submitted_at":"2025-02-20T05:41:55Z","title":"PC-Agent: A Hierarchical Multi-Agent Collaboration Framework for Complex Task Automation on PC","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.14282","snapshot_observed_at":"2026-08-07T06:00:19.328784Z","title":"Pc-agent: A hierarchical multi-agent collaboration framework for complex task automation on pc,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":66,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.328784Z"},"links":{"cited_paper":"/paper/2502.14282","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:64731336c307bfa30ebe660bb0776285eae5544f8e209560afaff808abfc4b3f","observation_id":"6c4ce685-e80a-427b-8709-453443680570","resolution":{"observed_at":"2026-08-07T06:00:19.328784Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.346134Z","title":"Androidinthewild: A large-scale dataset for android device control,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":67,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.346134Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:7902b98f5da487f7df18837e136b5662089eeb5423b9e89d13af3a00d227370a","observation_id":"c24c036c-e0ba-44ec-92ac-ba87eb081604","resolution":{"observed_at":"2026-08-07T06:00:19.346134Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.07615","last_updated":"2023-05-23T16:08:10Z","snapshot_observed_at":"2026-08-11T22:51:17.027870Z","submitted_at":"2022-11-14T18:36:19Z","title":"UGIF: UI Grounded Instruction Following","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.07615","snapshot_observed_at":"2026-08-07T06:00:19.337239Z","title":"Ugif: Ui grounded instruction following,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":68,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.337239Z"},"links":{"cited_paper":"/paper/2211.07615","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:3933cf92ff7a0721de2612dd6ae0ff7e1e3402885b49120f874d306718e712bc","observation_id":"53c92253-b41c-4aeb-bb37-9ff2de041d01","resolution":{"observed_at":"2026-08-07T06:00:19.337239Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2104.08560","last_updated":"2021-04-17T14:48:02Z","snapshot_observed_at":"2026-08-11T00:42:37.521754Z","submitted_at":"2021-04-17T14:48:02Z","title":"Mobile App Tasks with Iterative Feedback (MoTIF): Addressing Task Feasibility in Interactive Visual Environments","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.08560","snapshot_observed_at":"2026-08-07T06:00:19.341889Z","title":"Mobile app tasks with iterative feedback (motif): Addressing task feasibil- ity in interactive visual environments,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":69,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.341889Z"},"links":{"cited_paper":"/paper/2104.08560","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:6fad702215da603ff6bd99e3e6700b2ac8d53098c0eecccdc95cb8eaa2a541f4","observation_id":"b16d9ff5-45a8-41e3-84d9-6a62c76dd8ad","resolution":{"observed_at":"2026-08-07T06:00:19.341889Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.16660","last_updated":"2025-07-21T02:55:09Z","snapshot_observed_at":"2026-08-04T12:26:21.610975Z","submitted_at":"2024-04-25T14:56:32Z","title":"Benchmarking Mobile Device Control Agents across Diverse Configurations","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.16660","snapshot_observed_at":"2026-08-07T06:00:19.358520Z","title":"Benchmarking mobile device control agents across diverse configurations,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":70,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.358520Z"},"links":{"cited_paper":"/paper/2404.16660","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:bfce3fc91c4b15dc75f0fc6924b92075a38450dced2d958bf67736461612273b","observation_id":"c516cf87-86ac-4af6-ad50-884687aaba0e","resolution":{"observed_at":"2026-08-07T06:00:19.358520Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.349977Z","title":"On the effects of data scale on ui control agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":71,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.349977Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:9a3b2ccc1bf26d0115840d8187e467065c8c2c72f06aba791c19dbd15233b3e3","observation_id":"c35fdbb1-a917-4fae-82f7-bb9d1f4e8f26","resolution":{"observed_at":"2026-08-07T06:00:19.349977Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.17490","last_updated":"2025-05-29T02:43:50Z","snapshot_observed_at":"2026-08-11T03:51:51.000322Z","submitted_at":"2024-07-03T17:59:58Z","title":"AMEX: Android Multi-annotation Expo Dataset for Mobile GUI Agents","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.17490","snapshot_observed_at":"2026-08-07T06:00:19.353842Z","title":"Amex: Android multi- annotation expo dataset for mobile gui agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":72,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.353842Z"},"links":{"cited_paper":"/paper/2407.17490","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:44cec60a1d2a61bf4431f3e004b38a98fc361f6c406aa2076060ec64b311477f","observation_id":"597d503c-9e1f-4749-b85a-2961a22930c9","resolution":{"observed_at":"2026-08-07T06:00:19.353842Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.371285Z","title":"A3: Android agent arena for mobile gui agents,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":73,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.371285Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:4d314fa3b8378e6ffcfd8a8b6870ead22119a005c107d042b889830e18a4f396","observation_id":"662a50e3-fee9-4a80-a395-7d688686b468","resolution":{"observed_at":"2026-08-07T06:00:19.371285Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.14573","last_updated":"2025-04-06T20:37:50Z","snapshot_observed_at":"2026-08-12T07:38:32.454201Z","submitted_at":"2024-05-23T13:48:54Z","title":"AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.14573","snapshot_observed_at":"2026-08-07T06:00:19.362479Z","title":"Androidworld: A dynamic benchmarking environment for autonomous agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":74,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.362479Z"},"links":{"cited_paper":"/paper/2405.14573","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:38b468e4c65cfc46c7de28e2c6cd38ad040e579025279796f657e3afe4c1d069","observation_id":"d43916d4-6038-4196-9424-1085d5bd84e3","resolution":{"observed_at":"2026-08-07T06:00:19.362479Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.00993","last_updated":"2024-07-01T06:10:01Z","snapshot_observed_at":"2026-08-09T18:34:56.808764Z","submitted_at":"2024-07-01T06:10:01Z","title":"Mobile-Bench: An Evaluation Benchmark for LLM-based Mobile Agents","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.00993","snapshot_observed_at":"2026-08-07T06:00:19.366743Z","title":"Mobile-bench: An evaluation benchmark for llm-based mobile agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":75,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.366743Z"},"links":{"cited_paper":"/paper/2407.00993","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:b0010a4321dad5eb59f0e3d1dfaf95d710c1f4651c1cd6843a305cbe86834e1a","observation_id":"8946c67a-a648-4d4d-a8b3-ed1c6b20f1df","resolution":{"observed_at":"2026-08-07T06:00:19.366743Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.384014Z","title":"Mobilesafetybench: Evaluating safety of autonomous agents in mobile device control,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":76,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.384014Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:65e12293e5284a0e3d9702958f18d1f92a8f70be199f9e5d0068cc4e8cfdeaf0","observation_id":"7b013bde-7dff-41c4-a047-955f874f7890","resolution":{"observed_at":"2026-08-07T06:00:19.384014Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.08144","last_updated":"2024-06-13T11:51:37Z","snapshot_observed_at":"2026-07-06T15:26:58.583640Z","submitted_at":"2023-05-14T12:31:03Z","title":"Mobile-Env: Building Qualified Evaluation Benchmarks for LLM-GUI Interaction","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.08144","snapshot_observed_at":"2026-08-07T06:00:19.375445Z","title":"Mobile-env: Building qualified evaluation benchmarks for llm-gui interaction,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":77,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.375445Z"},"links":{"cited_paper":"/paper/2305.08144","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:4b02c31e705c7b14d34a0230b4356068562cbecf8f45e841dd166b776ed4169a","observation_id":"61b6aa15-c081-4984-9703-32da31ddb09a","resolution":{"observed_at":"2026-08-07T06:00:19.375445Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.379903Z","title":"Understanding the weakness of large language model agents within a complex android environment,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":78,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.379903Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:3f65a967fae30f0a231f935acfe54744dcf1a22c6ce51289492e2a6f123b4d70","observation_id":"880efb3e-e422-4412-8589-7df047937b48","resolution":{"observed_at":"2026-08-07T06:00:19.379903Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.396863Z","title":"Llamatouch: A faithful and scalable testbed for mobile ui task automation,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":79,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.396863Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:761131497697196a4526e2634d10a3cb5064ca537d5e248fa4e18e97dfd4f584","observation_id":"e51160c5-3605-4bfa-a3ea-9c04408c3173","resolution":{"observed_at":"2026-08-07T06:00:19.396863Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.08184","last_updated":"2024-06-12T13:14:50Z","snapshot_observed_at":"2026-07-06T18:29:34.538901Z","submitted_at":"2024-06-12T13:14:50Z","title":"MobileAgentBench: An Efficient and User-Friendly Benchmark for Mobile LLM Agents","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.08184","snapshot_observed_at":"2026-08-07T06:00:19.388589Z","title":"Mobileagentbench: An efficient and user-friendly benchmark for mobile llm agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":80,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.388589Z"},"links":{"cited_paper":"/paper/2406.08184","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:69648b8a917c7e25675221b8d9ceac19465f19dc53982bf23ea98d6e7a4bdb1c","observation_id":"8277fa0c-d3c7-465f-bcbe-5fb718485389","resolution":{"observed_at":"2026-08-07T06:00:19.388589Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.392685Z","title":"Spa-bench: A comprehensive benchmark for smartphone agent evaluation,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":81,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.392685Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:0e0db14ae7aacbeee95fb35a9269ef1b82416e0db5d19c234be21cef9a5e127d","observation_id":"b46f2ab7-81a9-45b5-9190-efb3a3381d3d","resolution":{"observed_at":"2026-08-07T06:00:19.392685Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1803.05457","last_updated":"2018-03-14T18:04:21Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2018-03-14T18:04:21Z","title":"Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1803.05457","snapshot_observed_at":"2026-08-07T06:00:19.409750Z","title":"Think you have solved question answering? try arc, the ai2 reasoning challenge,","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":82,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.409750Z"},"links":{"cited_paper":"/paper/1803.05457","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:14d35cce6274a211b8c63e64df980fa35e3ee2814a879ca1a187b508c84f6555","observation_id":"b9249483-3387-4fc9-807e-6b48abb01468","resolution":{"observed_at":"2026-08-07T06:00:19.409750Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.24024","last_updated":"2024-11-04T05:57:31Z","snapshot_observed_at":"2026-08-11T02:30:15.935322Z","submitted_at":"2024-10-31T15:25:20Z","title":"AndroidLab: Training and Systematic Benchmarking of Android Autonomous Agents","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.24024","snapshot_observed_at":"2026-08-07T06:00:19.401380Z","title":"Androidlab: Training and sys- tematic benchmarking of android autonomous agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":83,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.401380Z"},"links":{"cited_paper":"/paper/2410.24024","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:8855c5a1b29c0f1464425c755bd5d83a0e34e2d11c25b22b3a43494b260abe31","observation_id":"fb119f8e-8e28-43f3-8f4a-0b5c711da058","resolution":{"observed_at":"2026-08-07T06:00:19.401380Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.405415Z","title":"Autoeval: A practical framework for autonomous evaluation of mobile agents,","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":84,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.405415Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:d9325584199af07bee77e034210eb962480f492381bc5b530cfe3b08dd1d0a4a","observation_id":"15d98393-a742-4dc2-aff9-a6ba44f84d3d","resolution":{"observed_at":"2026-08-07T06:00:19.405415Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.422789Z","title":"Qasa: advanced question answer- ing on scientific articles,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":85,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.422789Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:6cf3c0ef6243873b90d525bff346a1d9f72e5a7510f49e3dbb66396a47d4c89a","observation_id":"0fdde1d4-9b54-4d4e-b018-1050ccf0519e","resolution":{"observed_at":"2026-08-07T06:00:19.422789Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2209.09513","last_updated":"2022-10-17T07:46:47Z","snapshot_observed_at":"2026-08-02T03:10:19.073297Z","submitted_at":"2022-09-20T07:04:24Z","title":"Learn to Explain: Multimodal Reasoning via Thought Chains for Science Question Answering","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2209.09513","snapshot_observed_at":"2026-08-07T06:00:19.414057Z","title":"Learn to explain: Multimodal reasoning via thought chains for science question answering,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":86,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.414057Z"},"links":{"cited_paper":"/paper/2209.09513","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:7d704e43e9a51fb3c0449f26774e32e658b5d29c94f8d295a5c2a178715f6c16","observation_id":"791bbd92-ebc3-44b0-beff-a12e5c7f077b","resolution":{"observed_at":"2026-08-07T06:00:19.414057Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2105.03011","last_updated":"2021-05-07T00:12:34Z","snapshot_observed_at":"2026-08-11T00:52:16.498491Z","submitted_at":"2021-05-07T00:12:34Z","title":"A Dataset of Information-Seeking Questions and Answers Anchored in Research Papers","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2105.03011","snapshot_observed_at":"2026-08-07T06:00:19.418439Z","title":"A dataset of information-seeking questions and answers anchored in research papers,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":87,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.418439Z"},"links":{"cited_paper":"/paper/2105.03011","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:6db97b21ff57c6952867e8846f92c3eeb852566185b8127e63446c471647d5e0","observation_id":"e72e4600-702a-421d-b460-bf671fe7bd72","resolution":{"observed_at":"2026-08-07T06:00:19.418439Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2104.06486","last_updated":"2021-11-23T01:12:57Z","snapshot_observed_at":"2026-08-09T20:11:46.800890Z","submitted_at":"2021-04-13T19:59:34Z","title":"MS2: Multi-Document Summarization of Medical Studies","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.06486","snapshot_observed_at":"2026-08-07T06:00:19.435832Z","title":"Ms2: Multi-document summarization of medical studies,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":88,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.435832Z"},"links":{"cited_paper":"/paper/2104.06486","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:c00f0d8719c2983c662cb68f15ce05aef0b1f5076c79dcedad1a588717280aa2","observation_id":"3ac21333-8046-40ab-83e6-38322acfbae1","resolution":{"observed_at":"2026-08-07T06:00:19.435832Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.20477","last_updated":"2024-06-04T16:03:57Z","snapshot_observed_at":"2026-07-06T18:23:01.681030Z","submitted_at":"2024-05-30T20:56:41Z","title":"Automated Focused Feedback Generation for Scientific Writing Assistance","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.20477","snapshot_observed_at":"2026-08-07T06:00:19.426701Z","title":"Auto- mated focused feedback generation for scientific writing assistance,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":89,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.426701Z"},"links":{"cited_paper":"/paper/2405.20477","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:5cfc36d48815b3f65a2da54123762283f194782020123aec07be8433c3dc8dc7","observation_id":"da3053c0-5c76-460c-9b66-332a6e24514e","resolution":{"observed_at":"2026-08-07T06:00:19.426701Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.22394","last_updated":"2025-05-25T06:54:06Z","snapshot_observed_at":"2026-08-11T18:37:09.377480Z","submitted_at":"2024-10-29T17:58:29Z","title":"AAAR-1.0: Assessing AI's Potential to Assist Research","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.22394","snapshot_observed_at":"2026-08-07T06:00:19.431100Z","title":"Aaar-1.0: Assessing ai’s potential to assist research,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":90,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.431100Z"},"links":{"cited_paper":"/paper/2410.22394","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:88108cacbeebb75ceddd83578ea44cf68ef4931b5f3adb38fb30a0af52a3fa6f","observation_id":"028d9b46-f4d3-4629-9976-e3ae1e18f5ef","resolution":{"observed_at":"2026-08-07T06:00:19.431100Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.04109","last_updated":"2024-09-06T08:25:03Z","snapshot_observed_at":"2026-08-08T00:38:42.440129Z","submitted_at":"2024-09-06T08:25:03Z","title":"Can LLMs Generate Novel Research Ideas? A Large-Scale Human Study with 100+ NLP Researchers","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.04109","snapshot_observed_at":"2026-08-07T06:00:19.449455Z","title":"Can llms generate novel research ideas? a large-scale human study with 100+ nlp researchers,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":91,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.449455Z"},"links":{"cited_paper":"/paper/2409.04109","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:71596b3df74e620721518787ea0fdd668974b94c27125dc57013c89107e5cd16","observation_id":"c1ca246e-b03b-465f-9f0b-cfa9a805e722","resolution":{"observed_at":"2026-08-07T06:00:19.449455Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.10362","last_updated":"2024-07-17T17:28:36Z","snapshot_observed_at":"2026-08-12T00:26:27.857559Z","submitted_at":"2024-07-14T23:52:25Z","title":"LAB-Bench: Measuring Capabilities of Language Models for Biology Research","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.10362","snapshot_observed_at":"2026-08-07T06:00:19.439930Z","title":"Lab-bench: Measur- ing capabilities of language models for biology research,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":92,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.439930Z"},"links":{"cited_paper":"/paper/2407.10362","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:903041d4a6e876c74f24549b30b1d77614d1c690bbf329f93579dff52f8f831a","observation_id":"03d5c5a1-2f1b-4f20-af18-7af2816ab996","resolution":{"observed_at":"2026-08-07T06:00:19.439930Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.10291","last_updated":"2025-09-07T23:37:18Z","snapshot_observed_at":"2026-08-12T08:03:29.062155Z","submitted_at":"2024-06-13T03:26:30Z","title":"ResearchArena: Benchmarking Large Language Models' Ability to Collect and Organize Information as Research Agents","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.10291","snapshot_observed_at":"2026-08-07T06:00:19.444970Z","title":"Researcharena: Benchmarking llms’ ability to collect and organize information as research agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":93,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.444970Z"},"links":{"cited_paper":"/paper/2406.10291","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:e67df0840ea94dbcbd10f58a7856c935dc72ac4744ec0d5451aa97bfaa99ec1d","observation_id":"bb95e92e-a272-40c9-b8f7-2e872a71d6f5","resolution":{"observed_at":"2026-08-07T06:00:19.444970Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.06769","last_updated":"2024-10-07T20:19:15Z","snapshot_observed_at":"2026-08-10T20:43:42.874538Z","submitted_at":"2024-06-10T20:08:44Z","title":"DISCOVERYWORLD: A Virtual Environment for Developing and Evaluating Automated Scientific Discovery Agents","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.06769","snapshot_observed_at":"2026-08-07T06:00:19.463504Z","title":"Discoveryworld: A virtual environment for developing and evaluating automated scientific discovery agents, 2024,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":94,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.463504Z"},"links":{"cited_paper":"/paper/2406.06769","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:9f0d824cc2180634d185f247b3431cfd38c8e0ea005188454a17ea73093faa85","observation_id":"49e1a027-28d6-459e-96e5-bb96895989a5","resolution":{"observed_at":"2026-08-07T06:00:19.463504Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.07559","last_updated":"2023-12-14T19:40:04Z","snapshot_observed_at":"2026-08-10T13:25:30.077534Z","submitted_at":"2023-12-08T18:50:20Z","title":"PaperQA: Retrieval-Augmented Generative Agent for Scientific Research","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.07559","snapshot_observed_at":"2026-08-07T06:00:19.453993Z","title":"Paperqa: Retrieval- augmented generative agent for scientific research,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":95,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.453993Z"},"links":{"cited_paper":"/paper/2312.07559","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:d356104d832ec7b38029574d43a9047cccb56c0dfc62724f4da15aa6a1320b92","observation_id":"fec61084-7634-407c-97d2-09d132ee4d14","resolution":{"observed_at":"2026-08-07T06:00:19.453993Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2203.07540","last_updated":"2022-11-14T17:52:27Z","snapshot_observed_at":"2026-08-06T07:25:42.562786Z","submitted_at":"2022-03-14T22:52:34Z","title":"ScienceWorld: Is your Agent Smarter than a 5th Grader?","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2203.07540","snapshot_observed_at":"2026-08-07T06:00:19.458614Z","title":"Scienceworld: Is your agent smarter than a 5th grader?, 2022,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":96,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.458614Z"},"links":{"cited_paper":"/paper/2203.07540","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:e3b814fb1e183d3a96d1115f3aea9c31896d133f647bd9665420c91c5dac49e6","observation_id":"27b08d10-0e8a-46ed-84aa-a9b8314678d3","resolution":{"observed_at":"2026-08-07T06:00:19.458614Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T06:00:19.476908Z","title":"Mlagent- bench: Evaluating language agents on machine learning experimentation, 2024,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":97,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.476908Z"},"links":{"citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:17abf0fc7d9394eb3880fa240d5396f2f1c24e779966980dacee9cbb963909a2","observation_id":"043cd23b-d07a-4272-8573-e9748501667f","resolution":{"observed_at":"2026-08-07T06:00:19.476908Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.05080","last_updated":"2025-03-31T14:39:44Z","snapshot_observed_at":"2026-07-06T19:29:05.492290Z","submitted_at":"2024-10-07T14:33:50Z","title":"ScienceAgentBench: Toward Rigorous Assessment of Language Agents for Data-Driven Scientific Discovery","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.05080","snapshot_observed_at":"2026-08-07T06:00:19.467715Z","title":"Scienceagentbench: Toward rigorous assessment of language agents for data-driven scientific discovery,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":98,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.467715Z"},"links":{"cited_paper":"/paper/2410.05080","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:15a4bd3056c734ae15dad1aa2a5a9383d8878cd6fcff6d237d738e1717a84b27","observation_id":"6a271e68-1308-4909-8500-8452e86d43b9","resolution":{"observed_at":"2026-08-07T06:00:19.467715Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.14499","last_updated":"2025-02-20T12:28:23Z","snapshot_observed_at":"2026-08-08T19:15:34.002979Z","submitted_at":"2025-02-20T12:28:23Z","title":"MLGym: A New Framework and Benchmark for Advancing AI Research Agents","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.14499","snapshot_observed_at":"2026-08-07T06:00:19.472486Z","title":"Mlgym: A new framework JOURNAL OF LATEX CLASS FILES, VOL. 14, NO. 8, AUGUST 2015 17 and benchmark for advancing ai research agents,","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":99,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.472486Z"},"links":{"cited_paper":"/paper/2502.14499","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:e28f09f863be8a56044790cc07ba81c82419c69a6fc4582947d874d75a2d0d18","observation_id":"79bccf71-c36d-4f0c-ba6c-fac4d942ced4","resolution":{"observed_at":"2026-08-07T06:00:19.472486Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.07331","last_updated":"2024-10-11T00:53:05Z","snapshot_observed_at":"2026-08-11T13:07:52.654284Z","submitted_at":"2024-10-09T18:00:05Z","title":"DA-Code: Agent Data Science Code Generation Benchmark for Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.07331","snapshot_observed_at":"2026-08-07T06:00:19.491104Z","title":"Da-code: Agent data science code generation benchmark for large language models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey","version":1},"reference_index":100,"source":"pdf_text","source_observed_at":"2026-08-07T06:00:19.491104Z"},"links":{"cited_paper":"/paper/2410.07331","citing_paper":"/paper/2506.11102"},"observation_digest":"sha256:73e40a0928cec0bd1711688239724a76d223ffce1fd4a0cda7e7134b6123078a","observation_id":"3b944511-a60a-467d-87f4-4285fa1271aa","resolution":{"observed_at":"2026-08-07T06:00:19.491104Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2506.11102","last_updated":"2025-06-06T17:52:18Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-07T05:54:56.593167Z","submitted_at":"2025-06-06T17:52:18Z","title":"Evolutionary Perspectives on the Evaluation of LLM-Based AI Agents: A Comprehensive Survey"},"reference_resolution":{"displayed":100,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":100,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":209},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"thesis":"As of 12 August 2026, this Paper Citation Record lists 100 of 209 outbound references and 4 inbound Pith citation observations for arXiv:2506.11102."}