{"as_of":"2026-08-14T00:03:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:c3e3b31e530303e94e573d7eb1d3f7512522319a522abe741badc9ef303749fc","coverage":[{"denominator":143,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":100,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-06T12:44:21.813422Z","state":"measured"},{"denominator":106,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":106,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-13T06:32:02.005865+00:00","state":"measured"},{"denominator":6,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":6,"source":"paper_references, paper_reference_links","source_observed_at":"2026-07-12T01:59:52.218232Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-03T10:27:56.022361Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"cited_work":{"arxiv_id":"2507.21504","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2507.21504","snapshot_observed_at":"2026-07-03T10:27:56.022361Z","title":"Evaluation and benchmarking of LLM agents: A survey","venue":null,"work_id":"af80895a-d45c-407b-a5d7-872b1b97f735","year":2025},"citing_paper":{"arxiv_id":"2603.20380","last_updated":"2026-05-15T21:59:54Z","snapshot_observed_at":"2026-08-12T22:35:48.851549Z","submitted_at":"2026-03-20T18:00:09Z","title":"Herding CATs: ALARA for Agent Harness Engineering in Portable Composable Multi-Agent Teams","version":2},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-05-21T11:12:46.626382Z"},"links":{"cited_paper":"/paper/2507.21504","citing_paper":"/paper/2603.20380"},"observation_digest":"sha256:8dd24a50a1d6745fe15cad418cba5fa448fb3a7cc245f13e781a29cbfb12f156","observation_id":"502c01ef-29cc-448f-98a6-0fec0e76d9b2","resolution":{"observed_at":"2026-05-21T11:14:08.422394Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"cited_work":{"arxiv_id":"2507.21504","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2507.21504","snapshot_observed_at":"2026-07-03T10:27:56.022361Z","title":"Evaluation and benchmarking of LLM agents: A survey","venue":null,"work_id":"af80895a-d45c-407b-a5d7-872b1b97f735","year":2025},"citing_paper":{"arxiv_id":"2605.16282","last_updated":"2026-04-11T04:25:19Z","snapshot_observed_at":"2026-08-12T21:07:09.004725Z","submitted_at":"2026-04-11T04:25:19Z","title":"Taxonomy and Consistency Analysis of Safety Benchmarks for AI Agents","version":1},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-05-21T01:42:55.693115Z"},"links":{"cited_paper":"/paper/2507.21504","citing_paper":"/paper/2605.16282"},"observation_digest":"sha256:db8963d5a8f8bf95de56cad9de0d28865786450d484d78fc48814af9c685026f","observation_id":"b79c7602-7f7c-4d27-a357-45e108c435fa","resolution":{"observed_at":"2026-05-21T01:43:56.680558Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"cited_work":{"arxiv_id":"2507.21504","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2507.21504","snapshot_observed_at":"2026-07-03T10:27:56.022361Z","title":"Evaluation and benchmarking of LLM agents: A survey","venue":null,"work_id":"af80895a-d45c-407b-a5d7-872b1b97f735","year":2025},"citing_paper":{"arxiv_id":"2605.16508","last_updated":"2026-05-15T18:05:21Z","snapshot_observed_at":"2026-08-11T14:48:42.420078Z","submitted_at":"2026-05-15T18:05:21Z","title":"The Scaling Laws of Skills in LLM Agent Systems","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-05-20T18:10:08.737710Z"},"links":{"cited_paper":"/paper/2507.21504","citing_paper":"/paper/2605.16508"},"observation_digest":"sha256:845668cc7389c5c74f0531e627db7f3b382a7a9d2d1247fffef24180e23534e5","observation_id":"83459fa1-372a-4927-952a-000830629ab0","resolution":{"observed_at":"2026-05-20T18:13:37.607891Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"cited_work":{"arxiv_id":"2507.21504","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2507.21504","snapshot_observed_at":"2026-07-03T10:27:56.022361Z","title":"Evaluation and benchmarking of LLM agents: A survey","venue":null,"work_id":"af80895a-d45c-407b-a5d7-872b1b97f735","year":2025},"citing_paper":{"arxiv_id":"2606.07805","last_updated":"2026-06-05T19:33:58Z","snapshot_observed_at":"2026-08-03T01:44:22.181409Z","submitted_at":"2026-06-05T19:33:58Z","title":"Beyond Goodhart's Law: A Dynamic Benchmark for Evaluating Compliance in Multi-Agent Systems","version":1},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-06-27T21:53:37.616447Z"},"links":{"cited_paper":"/paper/2507.21504","citing_paper":"/paper/2606.07805"},"observation_digest":"sha256:47e2e4906f82d713b2c950e0eeb0bd85695bc6b3b33d540c28727721945ba551","observation_id":"9e9ad036-2431-4c7d-a36a-e36b80b5c2d7","resolution":{"observed_at":"2026-07-02T17:47:17.744554Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"cited_work":{"arxiv_id":"2507.21504","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2507.21504","snapshot_observed_at":"2026-07-03T10:27:56.022361Z","title":"Evaluation and benchmarking of LLM agents: A survey","venue":null,"work_id":"af80895a-d45c-407b-a5d7-872b1b97f735","year":2025},"citing_paper":{"arxiv_id":"2606.11686","last_updated":"2026-06-10T05:55:13Z","snapshot_observed_at":"2026-07-06T23:50:44.713490Z","submitted_at":"2026-06-10T05:55:13Z","title":"Layer-Isolated Evaluation: Gating the Deterministic Scaffold of a Production LLM Agent with a No-LLM, Regression-Locked Test Harness","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-06-27T10:04:29.791558Z"},"links":{"cited_paper":"/paper/2507.21504","citing_paper":"/paper/2606.11686"},"observation_digest":"sha256:9fdd1c4fa20f0417db6d1ac1ec1f7bdb738c5ee553ab7e0d9ff29111535cfb46","observation_id":"b0fd8271-1371-4f80-9afb-b8ec1bb1dc4e","resolution":{"observed_at":"2026-07-03T10:27:56.024402Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2507.21504","snapshot_observed_at":"2026-07-12T01:59:52.218232Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.03510","last_updated":"2026-07-03T17:37:10Z","snapshot_observed_at":"2026-08-06T15:36:16.441466Z","submitted_at":"2026-07-03T17:37:10Z","title":"CAGE-1: Control, Assurance, and Governance Evaluation for Enterprise Agentic AI","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-07-12T01:59:52.218232Z"},"links":{"cited_paper":"/paper/2507.21504","citing_paper":"/paper/2607.03510"},"observation_digest":"sha256:91d0bfbc84876f2a0cb9542623eec4ce4f0320ad50547032b5b07d2706f56bc4","observation_id":"6e863949-2b1e-4d52-90bb-120f43c84ea3","resolution":{"observed_at":"2026-07-12T01:59:52.218232Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2507.21504/citation-record","integrity":"/paper/2507.21504/integrity","json":"/paper/2507.21504/citation-record.json","paper":"/paper/2507.21504"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.482897Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.482897Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:52c80596ec1a2f8d5637f7a119bf3ac33961d8f466d420925f6b35097da867fa","observation_id":"01cdcaa9-3927-49bf-beb2-97395950eaba","resolution":{"observed_at":"2026-08-06T12:44:21.482897Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.486623Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.486623Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:d6350d35623d6dc1d50ec4a1679cdda872c1c1afe0f29a4d794bc7171863ddef","observation_id":"99349306-a237-44b4-b97b-f48a3ee2428a","resolution":{"observed_at":"2026-08-06T12:44:21.486623Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.489909Z","title":"2024.Inspect AI: Framework for Large Language Model Evaluations","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.489909Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:b2893b208ad0f8c7f9dd901e3c556de7b6c8b416aec6796c7e138ac1914129b6","observation_id":"d6b75862-6f1c-4a82-b403-cf6c41ea3fb6","resolution":{"observed_at":"2026-08-06T12:44:21.489909Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.493073Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.493073Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:cd09e4b8699ab5cbeb37f4486ebfd380a949775cdb17b785ee988aab0979f422","observation_id":"3acd0a5f-50e6-42ab-b9c6-5501b546d41d","resolution":{"observed_at":"2026-08-06T12:44:21.493073Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.495973Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.495973Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:7f38a2440adca3f7b47f41abb509f44fd188922e54281bc24eede6b683b473b1","observation_id":"a0183b9e-8c90-4431-93d2-6f8bc626cc76","resolution":{"observed_at":"2026-08-06T12:44:21.495973Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.01943","last_updated":"2024-06-04T03:54:53Z","snapshot_observed_at":"2026-08-12T23:50:55.710665Z","submitted_at":"2024-06-04T03:54:53Z","title":"Enhancing Trust in LLMs: Algorithms for Comparing and Interpreting LLMs","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.01943","snapshot_observed_at":"2026-08-06T12:44:21.502718Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.502718Z"},"links":{"cited_paper":"/paper/2406.01943","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:ca6bf208e70ab2137bf8aab2c33a35063642c26692f6e73eb99d7e56cc6aa613","observation_id":"3e084299-b3f2-4353-8702-1ac4c1ff8a31","resolution":{"observed_at":"2026-08-06T12:44:21.502718Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.07201","last_updated":"2023-08-14T15:13:04Z","snapshot_observed_at":"2026-08-02T01:34:38.978920Z","submitted_at":"2023-08-14T15:13:04Z","title":"ChatEval: Towards Better LLM-based Evaluators through Multi-Agent Debate","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.07201","snapshot_observed_at":"2026-08-06T12:44:21.506012Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.506012Z"},"links":{"cited_paper":"/paper/2308.07201","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:408facf5560898e14e34e10f02846af2e1483f1ecebddc26368e2f0dce33cb32","observation_id":"f046b40a-e071-4cab-b409-b5d75cd66385","resolution":{"observed_at":"2026-08-06T12:44:21.506012Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.509324Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.509324Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:2ecebe36e22953f9fd8e6e08e7f4ca3d52d52ae8d7465c009e9070ba7f083ec2","observation_id":"6b32d450-e913-47cb-8d89-384fcf71762e","resolution":{"observed_at":"2026-08-06T12:44:21.509324Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.512128Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.512128Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:17fe6d3863c1ec5c8777a9536ad0a89b7b9b76e065c1255cd4b3e52cfb1be7de","observation_id":"e0ce98c8-6a51-450f-a48f-dfd9539403e7","resolution":{"observed_at":"2026-08-06T12:44:21.512128Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.517960Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.517960Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:96e6b7776d39c076c8d1fd8acd51c2bdf7955bc1225abe8608d98d45daf37896","observation_id":"a5fc4b80-d188-49ad-920f-98e5b6681c0a","resolution":{"observed_at":"2026-08-06T12:44:21.517960Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.05080","last_updated":"2025-03-31T14:39:44Z","snapshot_observed_at":"2026-08-12T22:29:07.248687Z","submitted_at":"2024-10-07T14:33:50Z","title":"ScienceAgentBench: Toward Rigorous Assessment of Language Agents for Data-Driven Scientific Discovery","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.05080","snapshot_observed_at":"2026-08-06T12:44:21.520560Z","title":"Baker, Benjamin Burns, Daniel Adu-Ampratwum, Xuhui Huang, Xia Ning, Song Gao, Yu Su, and Huan Sun","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.520560Z"},"links":{"cited_paper":"/paper/2410.05080","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:71d08a64b869701e12f06a4a4e230153e728d584bd82e94abd7ef0cca77741aa","observation_id":"2094d729-a247-4398-969c-1bfd1da0a94a","resolution":{"observed_at":"2026-08-06T12:44:21.520560Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.523530Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.523530Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:59926a458ce33a3ebc326767baf808477e54388b74a5b99d9f225c726a0bab2c","observation_id":"724229a0-0d2e-477b-bc0a-f9fcc9135313","resolution":{"observed_at":"2026-08-06T12:44:21.523530Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.12784","last_updated":"2024-07-17T17:59:47Z","snapshot_observed_at":"2026-08-12T23:20:07.101645Z","submitted_at":"2024-07-17T17:59:47Z","title":"AgentPoison: Red-teaming LLM Agents via Poisoning Memory or Knowledge Bases","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.12784","snapshot_observed_at":"2026-08-06T12:44:21.530025Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.530025Z"},"links":{"cited_paper":"/paper/2407.12784","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:4f26426f13af85e6d5e6080bd966e8c3f1ac54b5fa93f514874d9f6efeeb1ed6","observation_id":"33451500-2922-4100-8d9f-25563c283b73","resolution":{"observed_at":"2026-08-06T12:44:21.530025Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.05467","last_updated":"2025-02-28T16:02:27Z","snapshot_observed_at":"2026-08-13T05:20:59.084759Z","submitted_at":"2024-12-06T23:43:59Z","title":"The BrowserGym Ecosystem for Web Agent Research","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.05467","snapshot_observed_at":"2026-08-06T12:44:21.532807Z","title":"Xu, Siva Reddy, Quentin Cappart, Graham Neubig, Ruslan Salakhutdinov, Nico- las Chapados, and Alexandre Lacoste","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.532807Z"},"links":{"cited_paper":"/paper/2412.05467","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:18141bdaeb41c3cb2efb4d90bb90362b2dc6cfcda968e99bbac0e6d46045718e","observation_id":"ae999c9c-d449-48de-a343-0099d907a1c3","resolution":{"observed_at":"2026-08-06T12:44:21.532807Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.14033","last_updated":"2024-01-15T03:18:25Z","snapshot_observed_at":"2026-08-13T20:44:12.910386Z","submitted_at":"2023-12-21T17:02:06Z","title":"T-Eval: Evaluating the Tool Utilization Capability of Large Language Models Step by Step","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.14033","snapshot_observed_at":"2026-08-06T12:44:21.526623Z","title":"doi:10.48550/arXiv.2312.14033","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.526623Z"},"links":{"cited_paper":"/paper/2312.14033","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:c2c2b7cefa3f110b034ff54bc84b2239ddc096f93d50df7981345249fb9e4832","observation_id":"af3d1478-45bd-4d65-b7e9-7e7899f8da05","resolution":{"observed_at":"2026-08-06T12:44:21.526623Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.08881","last_updated":"2025-05-16T13:05:27Z","snapshot_observed_at":"2026-08-12T22:14:54.279198Z","submitted_at":"2024-10-25T20:17:59Z","title":"Can We Trust AI Agents? A Case Study of an LLM-Based Multi-Agent System for Ethical AI","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.08881","snapshot_observed_at":"2026-08-06T12:44:21.538622Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.538622Z"},"links":{"cited_paper":"/paper/2411.08881","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:9550faf61fd639cc0276031f5293eb9ab70d04278b730d10d9d6502c1ddd58f1","observation_id":"e3010720-da73-41a1-9bf6-9021a3d30020","resolution":{"observed_at":"2026-08-06T12:44:21.538622Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.13352","last_updated":"2024-11-24T22:04:23Z","snapshot_observed_at":"2026-07-06T18:33:32.806635Z","submitted_at":"2024-06-19T08:55:56Z","title":"AgentDojo: A Dynamic Environment to Evaluate Prompt Injection Attacks and Defenses for LLM Agents","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.13352","snapshot_observed_at":"2026-08-06T12:44:21.541517Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.541517Z"},"links":{"cited_paper":"/paper/2406.13352","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:4201505ac4c849de8a0148b740d7509a6058ea504dcb07877c976348fc26817f","observation_id":"25d1dbed-283a-4454-84b1-19d51b7eb010","resolution":{"observed_at":"2026-08-06T12:44:21.541517Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.06613","last_updated":"2024-07-22T14:32:33Z","snapshot_observed_at":"2026-08-12T23:48:17.108802Z","submitted_at":"2024-06-07T00:28:43Z","title":"GameBench: Evaluating Strategic Reasoning Abilities of LLM Agents","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.06613","snapshot_observed_at":"2026-08-06T12:44:21.535787Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.535787Z"},"links":{"cited_paper":"/paper/2406.06613","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:06cbc90b5b2cba3419e80c91d0d88d427600cb2bb372b1aa43176d1ba5267412","observation_id":"fd5e71eb-3fd5-4fc8-bee5-f08a2d330abe","resolution":{"observed_at":"2026-08-06T12:44:21.535787Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.547500Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.547500Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:e1882643a3b3c0d115cf41c1ffcc702f4b76c7250976747b83cc109f2d63814e","observation_id":"f830591c-6c6c-4755-b8b5-a1772cfe8ebc","resolution":{"observed_at":"2026-08-06T12:44:21.547500Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.03568","last_updated":"2024-01-25T21:20:27Z","snapshot_observed_at":"2026-08-12T12:11:22.336410Z","submitted_at":"2024-01-07T19:11:18Z","title":"Agent AI: Surveying the Horizons of Multimodal Interaction","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.03568","snapshot_observed_at":"2026-08-06T12:44:21.553434Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.553434Z"},"links":{"cited_paper":"/paper/2401.03568","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:3931c4e3dfbd6875eaae11c531cf57aea0495c673a22e3afebdd5f0bd620b7f1","observation_id":"93374fe1-760a-482a-98e8-9082b5f5cfd0","resolution":{"observed_at":"2026-08-06T12:44:21.553434Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.00993","last_updated":"2024-07-01T06:10:01Z","snapshot_observed_at":"2026-08-12T23:31:36.028281Z","submitted_at":"2024-07-01T06:10:01Z","title":"Mobile-Bench: An Evaluation Benchmark for LLM-based Mobile Agents","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.00993","snapshot_observed_at":"2026-08-06T12:44:21.544451Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.544451Z"},"links":{"cited_paper":"/paper/2407.00993","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:454209c9f62e4e876e9fcb621971d261146c234772e25afdd490ea531d379665","observation_id":"33514e0b-4b21-431b-b16a-68848f3dad5e","resolution":{"observed_at":"2026-08-06T12:44:21.544451Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.08144","last_updated":"2024-04-17T04:34:39Z","snapshot_observed_at":"2026-08-12T18:31:19.542322Z","submitted_at":"2024-04-11T22:07:19Z","title":"LLM Agents can Autonomously Exploit One-day Vulnerabilities","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.08144","snapshot_observed_at":"2026-08-06T12:44:21.562169Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.562169Z"},"links":{"cited_paper":"/paper/2404.08144","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:b8dbdca220a948373d8b106b56b774fc57c2eba053d4520efea4f6c121ad7fbe","observation_id":"8c2992d1-6232-476b-b113-711a9cbf67ba","resolution":{"observed_at":"2026-08-06T12:44:21.562169Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.01495","last_updated":"2025-05-07T05:01:14Z","snapshot_observed_at":"2026-08-12T23:51:23.082136Z","submitted_at":"2024-06-03T16:21:38Z","title":"Re-ReST: Reflection-Reinforced Self-Training for Language Agents","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.01495","snapshot_observed_at":"2026-08-06T12:44:21.550610Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.550610Z"},"links":{"cited_paper":"/paper/2406.01495","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:ba24453a23d7fae6a4101a48d851edded833255a08509923db12c736d5343005","observation_id":"dac87e5d-7d37-4502-a571-c8eec1d99f71","resolution":{"observed_at":"2026-08-06T12:44:21.550610Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.568290Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.568290Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:6ec6c44380c74ad9d40994b0f8dad005745ddaccc68a9bce585152c27b269b5b","observation_id":"8f11e5aa-5671-45f3-8e40-c48c16c85aec","resolution":{"observed_at":"2026-08-06T12:44:21.568290Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.556360Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.556360Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:8e7aa74cbb071ccc6849f9b908baced86c2f8b2196b22c83df4140db576faf93","observation_id":"5f71b096-20f3-4f48-95de-22c4c408171b","resolution":{"observed_at":"2026-08-06T12:44:21.556360Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.15217","last_updated":"2025-04-28T05:09:12Z","snapshot_observed_at":"2026-08-13T16:22:14.977210Z","submitted_at":"2023-09-26T19:23:54Z","title":"Ragas: Automated Evaluation of Retrieval Augmented Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.15217","snapshot_observed_at":"2026-08-06T12:44:21.559149Z","title":"arXiv:2309.15217 [cs.CL] https://arxiv.org/abs/2309.15217","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.559149Z"},"links":{"cited_paper":"/paper/2309.15217","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:1b3cb202a0b9d49c121067956b590166f4c48a61afdc0b1991ea7eb2ce0108ab","observation_id":"0dd12506-e3ec-4b8b-bf1a-21442e31dc7d","resolution":{"observed_at":"2026-08-06T12:44:21.559149Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.11462","last_updated":"2020-09-25T20:22:26Z","snapshot_observed_at":"2026-08-09T05:10:26.091710Z","submitted_at":"2020-09-24T03:17:19Z","title":"RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.11462","snapshot_observed_at":"2026-08-06T12:44:21.577059Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.577059Z"},"links":{"cited_paper":"/paper/2009.11462","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:f7223b852699186ff4ba00e1fb24a2dc0cdd20a9c770bbeeefd41f1fd604ab5c","observation_id":"98fbabe5-a8ae-4a36-b3a1-d6b09064913c","resolution":{"observed_at":"2026-08-06T12:44:21.577059Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.04468","last_updated":"2024-11-07T06:36:19Z","snapshot_observed_at":"2026-08-13T07:44:55.410720Z","submitted_at":"2024-11-07T06:36:19Z","title":"Magentic-One: A Generalist Multi-Agent System for Solving Complex Tasks","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.04468","snapshot_observed_at":"2026-08-06T12:44:21.565142Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.565142Z"},"links":{"cited_paper":"/paper/2411.04468","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:41a8d849c6e23c6fab44226a10578a0c04e89f5ee11daeb9292452ea3a6192e2","observation_id":"11fba66a-4d87-4501-bf86-cb4f7ade44bc","resolution":{"observed_at":"2026-08-06T12:44:21.565142Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.583195Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.583195Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:11c3dbb9d6ec7490eaa0ce2121fb3e19e39d27aeb71b01e6cad1be8917668afc","observation_id":"f4164373-270c-472f-a5c0-c1411bfdc6db","resolution":{"observed_at":"2026-08-06T12:44:21.583195Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.570992Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.570992Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:5d8a1c8ac0fc678412fd6de7d326eba0cb91d93bf17dbbacfc122f3e5fdfcf51","observation_id":"45062438-4da3-4cff-adb6-6ce4ec281027","resolution":{"observed_at":"2026-08-06T12:44:21.570992Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.09523","last_updated":"2024-11-14T15:40:04Z","snapshot_observed_at":"2026-08-13T23:02:17.490965Z","submitted_at":"2024-11-14T15:40:04Z","title":"Navigating the Risks: A Survey of Security, Privacy, and Ethics Threats in LLM-Based Agents","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.09523","snapshot_observed_at":"2026-08-06T12:44:21.574030Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.574030Z"},"links":{"cited_paper":"/paper/2411.09523","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:6f454ad575eb05aaacb5f492923c4a8e4dd8b4226bd80d39228d926dabb0b456","observation_id":"03ed874f-470c-43f2-a573-5a733ab8619a","resolution":{"observed_at":"2026-08-06T12:44:21.574030Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.01680","last_updated":"2024-04-19T01:15:16Z","snapshot_observed_at":"2026-08-10T13:10:07.804621Z","submitted_at":"2024-01-21T23:36:14Z","title":"Large Language Model based Multi-Agents: A Survey of Progress and Challenges","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.01680","snapshot_observed_at":"2026-08-06T12:44:21.591815Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.591815Z"},"links":{"cited_paper":"/paper/2402.01680","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:54cd4eb4a874bc3e5a80c4ce088ea3d1fd9ac601337d233a5017f80edb8f8853","observation_id":"3be0e896-3a10-443f-9f86-725f2db6b00d","resolution":{"observed_at":"2026-08-06T12:44:21.591815Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.06411","last_updated":"2024-04-09T16:01:24Z","snapshot_observed_at":"2026-08-13T00:34:04.098718Z","submitted_at":"2024-04-09T16:01:24Z","title":"AgentQuest: A Modular Benchmark Framework to Measure Progress and Improve LLM Agents","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.06411","snapshot_observed_at":"2026-08-06T12:44:21.580085Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.580085Z"},"links":{"cited_paper":"/paper/2404.06411","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:2977c7613892bd952e9c2da55389005655719aade11c122a5e41d2d444adba3b","observation_id":"c0640dad-e5f3-42c0-9f55-e0a2bed5c23e","resolution":{"observed_at":"2026-08-06T12:44:21.580085Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.597845Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.597845Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:a76f7164943b4ad27a815cb71088f8009ba2146d48d9b2454a1ef23f5a0ed667","observation_id":"d720d498-29d4-46c4-8d38-e7aed3ab185d","resolution":{"observed_at":"2026-08-06T12:44:21.597845Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.15594","last_updated":"2025-10-19T10:32:43Z","snapshot_observed_at":"2026-08-13T06:43:22.011336Z","submitted_at":"2024-11-23T16:03:35Z","title":"A Survey on LLM-as-a-Judge","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.15594","snapshot_observed_at":"2026-08-06T12:44:21.585641Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.585641Z"},"links":{"cited_paper":"/paper/2411.15594","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:c433258e7cb82d335c26204d76a9314afe6930b98468cf0b94aa1e6327c2b5ba","observation_id":"67888622-73c4-45c2-9b2e-e5bc28d3bc5c","resolution":{"observed_at":"2026-08-06T12:44:21.585641Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.588784Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.588784Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:23d4ba6bafa86179dcd53e11a264c80412b0e43040b099cecddf5b4ba06b5db3","observation_id":"57d265d4-3b1b-4bb5-b9cd-4067450ccb32","resolution":{"observed_at":"2026-08-06T12:44:21.588784Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.02716","last_updated":"2024-02-05T04:25:24Z","snapshot_observed_at":"2026-08-13T19:43:43.684798Z","submitted_at":"2024-02-05T04:25:24Z","title":"Understanding the planning of LLM agents: A survey","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.02716","snapshot_observed_at":"2026-08-06T12:44:21.606949Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.606949Z"},"links":{"cited_paper":"/paper/2402.02716","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:7e7d15298859d9a4a69998cc2a5b3897e290348d724a0f686d1ae27b605f163f","observation_id":"03971990-d79d-4f98-931d-b8d22cc98ff8","resolution":{"observed_at":"2026-08-06T12:44:21.606949Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.03578","last_updated":"2026-01-28T04:55:23Z","snapshot_observed_at":"2026-07-06T17:25:53.950578Z","submitted_at":"2024-02-05T23:06:42Z","title":"LLM Multi-Agent Systems: Challenges and Open Problems","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.03578","snapshot_observed_at":"2026-08-06T12:44:21.594720Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.594720Z"},"links":{"cited_paper":"/paper/2402.03578","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:bc9169cd076cd1a721c22ef3f82f1b0ceed2538bb21e9966ff0336aa84952899","observation_id":"1cf4dcc5-30b3-49f2-955f-a82ac905765a","resolution":{"observed_at":"2026-08-06T12:44:21.594720Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.612824Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.612824Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:eade91a7de6bbb2b97d6fdc9f9003027acb25ba1f9f142b576130663f4df9664","observation_id":"a47616f2-fdfc-43ee-8e73-50016babc9c6","resolution":{"observed_at":"2026-08-06T12:44:21.612824Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.02959","last_updated":"2024-09-21T14:49:53Z","snapshot_observed_at":"2026-08-13T04:03:04.928621Z","submitted_at":"2024-03-05T13:30:02Z","title":"AgentsCourt: Building Judicial Decision-Making Agents with Court Debate Simulation and Legal Knowledge Augmentation","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.02959","snapshot_observed_at":"2026-08-06T12:44:21.600583Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.600583Z"},"links":{"cited_paper":"/paper/2403.02959","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:365b5de84d0ee6a584cad428a4463f993ca901f0a409001b46ca2d8d77f9eaa9","observation_id":"7e4ffcf8-ac71-47ad-a8f1-b9f26e1d22c1","resolution":{"observed_at":"2026-08-06T12:44:21.600583Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.03300","last_updated":"2021-01-12T18:57:11Z","snapshot_observed_at":"2026-08-13T20:44:28.824685Z","submitted_at":"2020-09-07T17:59:25Z","title":"Measuring Massive Multitask Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.03300","snapshot_observed_at":"2026-08-06T12:44:21.603848Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.603848Z"},"links":{"cited_paper":"/paper/2009.03300","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:041e3a8f64d364bb6f3ac8f6b02bc0aab7e9fca5b9c916b705f9ab8a6ecf0bee","observation_id":"c0f909b1-34af-4a68-bfb1-59360d7225e9","resolution":{"observed_at":"2026-08-06T12:44:21.603848Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.624201Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.624201Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:2403740dd56811f60ef07adecc0c6c3914c622e408a7dacf97b0e78ad1317886","observation_id":"38e49dfc-b559-4e10-8a8e-db2336f3cace","resolution":{"observed_at":"2026-08-06T12:44:21.624201Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.03128","last_updated":"2024-12-04T19:49:02Z","snapshot_observed_at":"2026-08-13T19:44:16.428291Z","submitted_at":"2023-10-04T19:39:26Z","title":"MetaTool Benchmark for Large Language Models: Deciding Whether to Use Tools and Which to Use","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.03128","snapshot_observed_at":"2026-08-06T12:44:21.609937Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.609937Z"},"links":{"cited_paper":"/paper/2310.03128","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:cab3a74f67d1bbfc3d876993440266af2c70da47d997c6f590b5079a9b5b7b82","observation_id":"1576480d-e593-4139-8f1a-353f60d92f52","resolution":{"observed_at":"2026-08-06T12:44:21.609937Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.633063Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.633063Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:ff9cef2bf23f5ae6692cff5388b9489046c4463ca58b5823a0fb7e51cacbf881","observation_id":"f5195e60-8380-49c0-be77-aced989195e1","resolution":{"observed_at":"2026-08-06T12:44:21.633063Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.16294","last_updated":"2024-06-24T03:36:29Z","snapshot_observed_at":"2026-08-12T23:36:30.850943Z","submitted_at":"2024-06-24T03:36:29Z","title":"LangSuitE: Planning, Controlling and Interacting with Large Language Models in Embodied Text Environments","version":1},"cited_work":{"arxiv_id":"2406.16294","doi":null,"metadata_source":"pith","pith_arxiv_id":"2406.16294","snapshot_observed_at":"2026-08-06T12:44:22.814449Z","title":"LangSuitE: Planning, Controlling and Interacting with Large Language Models in Embodied Text Environments","venue":"cs.CL","work_id":"9088f36a-f945-4bf4-89af-6acf11aafe85","year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.615493Z"},"links":{"cited_paper":"/paper/2406.16294","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:e7f072b60c5ce7bf1de89463f6af4bddd714cacd43c671c2e184c1b53f7852c6","observation_id":"17afe007-792d-4036-81d8-4f6e86d4e29c","resolution":{"observed_at":"2026-08-06T12:44:22.818063Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.06770","last_updated":"2024-11-11T23:05:04Z","snapshot_observed_at":"2026-08-12T14:56:35.025839Z","submitted_at":"2023-10-10T16:47:29Z","title":"SWE-bench: Can Language Models Resolve Real-World GitHub Issues?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.06770","snapshot_observed_at":"2026-08-06T12:44:21.618485Z","title":"Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, and Karthik Narasimhan","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.618485Z"},"links":{"cited_paper":"/paper/2310.06770","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:030a872d442d98ae185f8b7324d9338652b16a9fbf9a0510be5ab2de657466c8","observation_id":"6fbcd1ef-99ba-44ae-9490-f3f8136fb2d4","resolution":{"observed_at":"2026-08-06T12:44:21.618485Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.621618Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.621618Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:b07dd1713ac98f19dc34edefa13f45bc00fb1c11443b19476a13b49ff2b663bf","observation_id":"b75ef5d4-56c0-4b5e-a026-a442f6ea8500","resolution":{"observed_at":"2026-08-06T12:44:21.621618Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.645335Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.645335Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:e962a01000a4c42c4589265033dcb7cefc783d411baae0fb22a4785ba8eb4f3c","observation_id":"fae8e12c-84d0-4d3a-8cf8-5b793eede1c1","resolution":{"observed_at":"2026-08-06T12:44:21.645335Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.13649","last_updated":"2024-06-06T02:01:09Z","snapshot_observed_at":"2026-08-05T16:01:54.847394Z","submitted_at":"2024-01-24T18:35:21Z","title":"VisualWebArena: Evaluating Multimodal Agents on Realistic Visual Web Tasks","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.13649","snapshot_observed_at":"2026-08-06T12:44:21.627127Z","title":"doi:10.48550/arXiv.2401.13649","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.627127Z"},"links":{"cited_paper":"/paper/2401.13649","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:a78b9058fba9a024482b3056af52f91884cc9f82eff13906d0f6ac7f4f47645c","observation_id":"95dbed4c-c632-46b2-847c-0137bac07bca","resolution":{"observed_at":"2026-08-06T12:44:21.627127Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2301.13298","last_updated":"2023-01-30T21:31:48Z","snapshot_observed_at":"2026-08-13T12:54:47.654946Z","submitted_at":"2023-01-30T21:31:48Z","title":"LongEval: Guidelines for Human Evaluation of Faithfulness in Long-form Summarization","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2301.13298","snapshot_observed_at":"2026-08-06T12:44:21.630164Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.630164Z"},"links":{"cited_paper":"/paper/2301.13298","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:19a8c0b2e8c5e0b74f0b3d8d8837656f7a4bb77ce83cceeab895c44293e327fd","observation_id":"08dd9ff0-baad-4333-b7dc-9e5ef96fc258","resolution":{"observed_at":"2026-08-06T12:44:21.630164Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.653693Z","title":"Manning, Christopher Ré, Diana Acosta-Navas, Drew A","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.653693Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:8c49e7c54638a8d0eacf3d230ddba87038dd131c47d8ccfd8910092d27c89719","observation_id":"988d73f9-f5c6-475f-830e-f6a5bc06d29f","resolution":{"observed_at":"2026-08-06T12:44:21.653693Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.11067","last_updated":"2025-01-19T14:58:35Z","snapshot_observed_at":"2026-08-11T15:38:18.564930Z","submitted_at":"2025-01-19T14:58:35Z","title":"IntellAgent: A Multi-Agent Framework for Evaluating Conversational AI Systems","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.11067","snapshot_observed_at":"2026-08-06T12:44:21.636050Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.636050Z"},"links":{"cited_paper":"/paper/2501.11067","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:360d41e11dc3b15a7e097d13bccaeb6130ca3bfba12240dcc8454ed2483d1c93","observation_id":"a7a1f55d-5608-463f-ba76-40eaeb9259b8","resolution":{"observed_at":"2026-08-06T12:44:21.636050Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.05579","last_updated":"2024-12-10T05:49:12Z","snapshot_observed_at":"2026-08-11T23:22:45.789386Z","submitted_at":"2024-12-07T08:07:24Z","title":"LLMs-as-Judges: A Comprehensive Survey on LLM-based Evaluation Methods","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.05579","snapshot_observed_at":"2026-08-06T12:44:21.639176Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.639176Z"},"links":{"cited_paper":"/paper/2412.05579","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:94ff25a7ef126e4a1a0cda7d7fc1065af73675a7bdfa6166672e7924fde3a6a0","observation_id":"1ea9c14a-43b7-48a2-9a75-5b196197df94","resolution":{"observed_at":"2026-08-06T12:44:21.639176Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2304.08244","last_updated":"2023-10-25T06:54:12Z","snapshot_observed_at":"2026-08-02T00:07:12.855748Z","submitted_at":"2023-04-14T14:05:32Z","title":"API-Bank: A Comprehensive Benchmark for Tool-Augmented LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2304.08244","snapshot_observed_at":"2026-08-06T12:44:21.642196Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.642196Z"},"links":{"cited_paper":"/paper/2304.08244","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:a863832bd550eeb0bce02abb97599aaabaf75f674b0ca89bca12af869cdcc1ed","observation_id":"4f5134ab-59be-4e29-8c6c-d1a72f2f2df9","resolution":{"observed_at":"2026-08-06T12:44:21.642196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.14928","last_updated":"2024-10-17T10:30:41Z","snapshot_observed_at":"2026-08-12T23:37:46.221159Z","submitted_at":"2024-06-21T07:37:19Z","title":"Autonomous Agents for Collaborative Task under Information Asymmetry","version":2},"cited_work":{"arxiv_id":"2406.14928","doi":null,"metadata_source":"pith","pith_arxiv_id":"2406.14928","snapshot_observed_at":"2026-08-06T12:44:22.759548Z","title":"Autonomous Agents for Collaborative Task under Information Asymmetry","venue":"cs.AI","work_id":"9924da1a-718d-4851-ab5b-946182fe8d27","year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.671369Z"},"links":{"cited_paper":"/paper/2406.14928","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:27e955ec427708553597199ca463275ba283617e026b71cf2c5a41f578ba353a","observation_id":"32246f49-b35c-4ff5-9d98-25867ca6336b","resolution":{"observed_at":"2026-08-06T12:44:22.763154Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.05459","last_updated":"2024-05-08T06:16:23Z","snapshot_observed_at":"2026-08-02T13:57:57.119489Z","submitted_at":"2024-01-10T09:25:45Z","title":"Personal LLM Agents: Insights and Survey about the Capability, Efficiency and Security","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.05459","snapshot_observed_at":"2026-08-06T12:44:21.648044Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.648044Z"},"links":{"cited_paper":"/paper/2401.05459","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:4d65bc5323d4c521b8c7238bf4a834873d76ff280d4fa7c292976417e40f4bfc","observation_id":"a16e74b8-bc20-4e6f-afc8-922a972f43e3","resolution":{"observed_at":"2026-08-06T12:44:21.648044Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.651084Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.651084Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:7ffd7693d891f3cbd161cf070b3002422fa416965ba14eb69240ea18b097268d","observation_id":"87cae1f6-0021-42e7-abc1-8c40a2a3919c","resolution":{"observed_at":"2026-08-06T12:44:21.651084Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.03688","last_updated":"2025-10-04T03:54:18Z","snapshot_observed_at":"2026-08-13T14:09:17.964744Z","submitted_at":"2023-08-07T16:08:11Z","title":"AgentBench: Evaluating LLMs as Agents","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.03688","snapshot_observed_at":"2026-08-06T12:44:21.679806Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.679806Z"},"links":{"cited_paper":"/paper/2308.03688","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:497d373ab6e8a0e51a96fd62e8587e015c4cb0d98e752b019187bc566fba48a8","observation_id":"5d0eda4b-5def-4aba-aad3-b03c122d49fe","resolution":{"observed_at":"2026-08-06T12:44:21.679806Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.682381Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.682381Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:787d89d0e60269b0bc6b9fe40ce4189631c50d356102c9ae0492151b28114bc7","observation_id":"4d83681e-34b9-4e2d-a9f5-7ea8018dcf15","resolution":{"observed_at":"2026-08-06T12:44:21.682381Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.659528Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.659528Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:ee01502d5bcdea093ad02d7379e41f12806d72257a35f0b27f12434ad6b2352f","observation_id":"10e97fee-f5b5-4387-a5b8-1543f55ffa76","resolution":{"observed_at":"2026-08-06T12:44:21.659528Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.04026","last_updated":"2023-08-08T03:59:28Z","snapshot_observed_at":"2026-08-13T18:07:56.522435Z","submitted_at":"2023-08-08T03:59:28Z","title":"AgentSims: An Open-Source Sandbox for Large Language Model Evaluation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.04026","snapshot_observed_at":"2026-08-06T12:44:21.662265Z","title":"arXiv preprint arXiv:2308.04026(2023)","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.662265Z"},"links":{"cited_paper":"/paper/2308.04026","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:40a5ccb1d8db9ebaa945eb227c36c41dc2ab390576f4db8c089c0e1ff9a8701d","observation_id":"acb862c6-b9b7-40ed-b0b5-1650e4f96031","resolution":{"observed_at":"2026-08-06T12:44:21.662265Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2024.findings-emnlp.267","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":null,"venue":null,"work_id":"55da9dd3-2a0a-44b3-80ff-d4d8012bf07c","year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.665219Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:db1248b0ef05ab1f370c79ac6ed787df3533c38af3dcb333968b12b4af1d3b9a","observation_id":"a71eb74e-e7d4-4f48-a9fc-14f70eaf0f26","resolution":{"observed_at":"2026-08-06T12:44:22.136258Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2504.01990","last_updated":"2025-08-02T12:44:02Z","snapshot_observed_at":"2026-08-12T16:45:40.094278Z","submitted_at":"2025-03-31T18:00:29Z","title":"Advances and Challenges in Foundation Agents: From Brain-Inspired Intelligence to Evolutionary, Collaborative, and Safe Systems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2504.01990","snapshot_observed_at":"2026-08-06T12:44:21.668218Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.668218Z"},"links":{"cited_paper":"/paper/2504.01990","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:f5637b8210865e2b8617741cbf151c84a9eaf7c40cdcabf41a9904321fb92e6c","observation_id":"b4988fd1-00f9-411c-a20b-cd5f66f6298d","resolution":{"observed_at":"2026-08-06T12:44:21.668218Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.13178","last_updated":"2024-12-23T20:12:48Z","snapshot_observed_at":"2026-08-13T07:41:55.930150Z","submitted_at":"2024-01-24T01:51:00Z","title":"AgentBoard: An Analytical Evaluation Board of Multi-turn LLM Agents","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.13178","snapshot_observed_at":"2026-08-06T12:44:21.699335Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":64,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.699335Z"},"links":{"cited_paper":"/paper/2401.13178","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:f0875486fc85ebd06ef215db17072b060f94f2d836a61b2d235f20b781a152ce","observation_id":"ff4c10d5-7239-4858-9037-39477c88bd91","resolution":{"observed_at":"2026-08-06T12:44:21.699335Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.674191Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":65,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.674191Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:5797976b00c51f28a4407fc14bca7076b0cb00dc6ba3d47a35f270879adc5ea8","observation_id":"4f007bf9-f524-4fe6-bddb-4a592f38fa06","resolution":{"observed_at":"2026-08-06T12:44:21.674191Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.03688","last_updated":"2025-10-04T03:54:18Z","snapshot_observed_at":"2026-08-13T14:09:17.964744Z","submitted_at":"2023-08-07T16:08:11Z","title":"AgentBench: Evaluating LLMs as Agents","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.03688","snapshot_observed_at":"2026-08-06T12:44:21.676884Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":66,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.676884Z"},"links":{"cited_paper":"/paper/2308.03688","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:f2e655193261f4d0c3091cc5029fea35c82bd679a0e7b5a720709eed3f5a14c8","observation_id":"8693fcb6-3f7d-4d62-ba10-250858c969b7","resolution":{"observed_at":"2026-08-06T12:44:21.676884Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.707889Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":67,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.707889Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:062398aac2a2bf64525b05a721e6b499073eb11a1aa2cd97b49712525ea1511e","observation_id":"5d6b6591-30e0-40d4-a330-824cbe460198","resolution":{"observed_at":"2026-08-06T12:44:21.707889Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.710615Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":68,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.710615Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:58acd8a3cd322825223bce951e17fc8e25d750123e3d2db82d83a1b0838861b2","observation_id":"a3bb5815-e184-4050-875e-aff77b02f5b8","resolution":{"observed_at":"2026-08-06T12:44:21.710615Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.09434","last_updated":"2023-05-16T13:46:52Z","snapshot_observed_at":"2026-08-13T11:41:02.842297Z","submitted_at":"2023-05-16T13:46:52Z","title":"Chatting with GPT-3 for Zero-Shot Human-Like Mobile Automated GUI Testing","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.09434","snapshot_observed_at":"2026-08-06T12:44:21.685043Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":69,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.685043Z"},"links":{"cited_paper":"/paper/2305.09434","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:5ecd93a723d6e88aecc8c8763579b4147fe060d7c53a644b6ac6f09ef0816146","observation_id":"e4833b80-cd87-46f3-804a-6d73e1bbc477","resolution":{"observed_at":"2026-08-06T12:44:21.685043Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.22394","last_updated":"2025-05-25T06:54:06Z","snapshot_observed_at":"2026-08-12T22:12:30.497386Z","submitted_at":"2024-10-29T17:58:29Z","title":"AAAR-1.0: Assessing AI's Potential to Assist Research","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.22394","snapshot_observed_at":"2026-08-06T12:44:21.687855Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":70,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.687855Z"},"links":{"cited_paper":"/paper/2410.22394","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:cf6eadf24f368dda6a919dd93a6326edecfd934f02e39a706ec952eb99f3f4cd","observation_id":"d7f7fd7c-d72d-4031-b562-f6a09f14307b","resolution":{"observed_at":"2026-08-06T12:44:21.687855Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.691267Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":71,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.691267Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:81c2d7d2007211738f6f6671be1230a1ba82771dbc3afc65b780c4828adf8746","observation_id":"332611bb-3bbf-433e-ab58-d8aa519a7dda","resolution":{"observed_at":"2026-08-06T12:44:21.691267Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.06292","last_updated":"2024-09-01T00:41:18Z","snapshot_observed_at":"2026-07-06T18:59:43.564435Z","submitted_at":"2024-08-12T16:58:11Z","title":"The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.06292","snapshot_observed_at":"2026-08-06T12:44:21.693952Z","title":"arXiv preprint arXiv:2408.06292(2024)","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":72,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.693952Z"},"links":{"cited_paper":"/paper/2408.06292","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:08d8bda60ab8ff2505f3cfe8d05a7bf722d97070eaf2394986e938be0643683a","observation_id":"51527181-da25-4aff-a12a-678c0932dc05","resolution":{"observed_at":"2026-08-06T12:44:21.693952Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.696779Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":73,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.696779Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:7ec24ddf13ec328cb73142f6a9a79106ccffc382a4f07e139de6fd642d921712","observation_id":"2793d074-519f-4448-b9f2-cfc2d738c4fa","resolution":{"observed_at":"2026-08-06T12:44:21.696779Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.730067Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":74,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.730067Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:e66957192c3416193c8a7e657c219508847b86197e8b01cad0e8c90d711a421c","observation_id":"d82f2c72-baa2-4505-9ab0-5d77ac44e58e","resolution":{"observed_at":"2026-08-06T12:44:21.730067Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.17753","last_updated":"2024-02-27T18:42:31Z","snapshot_observed_at":"2026-08-13T02:36:36.245972Z","submitted_at":"2024-02-27T18:42:31Z","title":"Evaluating Very Long-Term Conversational Memory of LLM Agents","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.17753","snapshot_observed_at":"2026-08-06T12:44:21.702317Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":75,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.702317Z"},"links":{"cited_paper":"/paper/2402.17753","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:148ebd5fa6b0dc29e34ffaec2835a40940e7a62900233735dc4747b40a6f4bae","observation_id":"ea8717b2-0ba8-4a3b-8d82-ba41f72e027e","resolution":{"observed_at":"2026-08-06T12:44:21.702317Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.705264Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":76,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.705264Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:3d483c800ba47cdfa0be9106c1257b65de40dba8c300c54cc82a15d8c5ad4223","observation_id":"8d34f258-59af-46a8-8de8-a4132c3b0ccb","resolution":{"observed_at":"2026-08-06T12:44:21.705264Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.23252","last_updated":"2025-03-08T23:37:49Z","snapshot_observed_at":"2026-08-13T22:18:47.339702Z","submitted_at":"2024-10-30T17:35:44Z","title":"Evaluating Cultural and Social Awareness of LLM Web Agents","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.23252","snapshot_observed_at":"2026-08-06T12:44:21.741655Z","title":"Fabbri, Divyansh Agarwal, Kung-Hsiang Huang, Sarah Tan, Nanyun Peng, and Chien-Sheng Wu","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":77,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.741655Z"},"links":{"cited_paper":"/paper/2410.23252","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:f4ef1266bc46fbefcdd2210cb6618dc744e3e46169e00a86e984f751868285d1","observation_id":"c47bb156-6098-4617-81f1-d02a36e835ab","resolution":{"observed_at":"2026-08-06T12:44:21.741655Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1806.03822","last_updated":"2018-06-11T06:10:11Z","snapshot_observed_at":"2026-08-10T03:45:00.871928Z","submitted_at":"2018-06-11T06:10:11Z","title":"Know What You Don't Know: Unanswerable Questions for SQuAD","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1806.03822","snapshot_observed_at":"2026-08-06T12:44:21.744477Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":78,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.744477Z"},"links":{"cited_paper":"/paper/1806.03822","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:e439ea30c1d498e3c3512c216e8b6b480528c24e656176bce3df50a21c62a3c1","observation_id":"f34adac2-6809-4442-b63e-0ee4f1d601d0","resolution":{"observed_at":"2026-08-06T12:44:21.744477Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.713486Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":79,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.713486Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:4fd092f1ddbfca7dfbcbf9e9740eb550124685d5df138384c608e8a64981934a","observation_id":"a63d1cf0-e7dd-4d36-8075-c300f73d5f27","resolution":{"observed_at":"2026-08-06T12:44:21.713486Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.716073Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":80,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.716073Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:b9dbb1b60fd52115ff73d25f10d9bca86f01135b07b2a28575f1631d125892a1","observation_id":"f021281d-6cf7-46de-9b3c-47488b8685eb","resolution":{"observed_at":"2026-08-06T12:44:21.716073Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.718892Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":81,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.718892Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:b87c69131573295672d33d91938adb253e6a03a910a6d58a095e8ca56acaf373","observation_id":"e4b2c4c2-a9cd-420d-b5c8-9c8bceee7f3f","resolution":{"observed_at":"2026-08-06T12:44:21.718892Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.721508Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":82,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.721508Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:884594aa6933207b8845403c42f6e8b730e47e1e1047cd45cdcc487ca25cea16","observation_id":"a32272ee-a3ea-4d59-bd10-7c943f08e3e2","resolution":{"observed_at":"2026-08-06T12:44:21.721508Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.13543","last_updated":"2025-04-01T14:45:22Z","snapshot_observed_at":"2026-08-13T22:17:36.984752Z","submitted_at":"2024-11-20T18:54:32Z","title":"BALROG: Benchmarking Agentic LLM and VLM Reasoning On Games","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.13543","snapshot_observed_at":"2026-08-06T12:44:21.724101Z","title":"arXiv:2411.13543 [cs.AI] https://arxiv.org/abs/2411.13543","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":83,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.724101Z"},"links":{"cited_paper":"/paper/2411.13543","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:f411206b53d514996b95e33875cfdd952aadce9dec18a7a85b7888ed411df9c6","observation_id":"40861379-0377-47af-9bf4-7b6d5b2f7118","resolution":{"observed_at":"2026-08-06T12:44:21.724101Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.12373","last_updated":"2024-07-16T06:19:21Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-06-18T07:58:33Z","title":"WebCanvas: Benchmarking Web Agents in Online Environments","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.12373","snapshot_observed_at":"2026-08-06T12:44:21.727061Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":84,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.727061Z"},"links":{"cited_paper":"/paper/2406.12373","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:c484c37683a85e6c6ba29ff7cc7913f93461753cb867f60b73e1621bdb88da8e","observation_id":"56cfd1f3-cdee-4b97-9412-567365435198","resolution":{"observed_at":"2026-08-06T12:44:21.727061Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.768017Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":85,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.768017Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:6f2831bb56160b2d8f6d3ef3960b61323282a2630906aaea23a93d422d1694f1","observation_id":"670d85e0-0722-4c56-8bcb-46e51cbff8b5","resolution":{"observed_at":"2026-08-06T12:44:21.768017Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.15334","last_updated":"2023-05-24T16:48:11Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-05-24T16:48:11Z","title":"Gorilla: Large Language Model Connected with Massive APIs","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.15334","snapshot_observed_at":"2026-08-06T12:44:21.732658Z","title":"Patil, Tianjun Zhang, Xin Wang, and Joseph E","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":86,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.732658Z"},"links":{"cited_paper":"/paper/2305.15334","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:cd8fed21ce6f962bd98a6481aceba2b70b4b9abef599798449d4484c743409a5","observation_id":"67ee1029-8780-44f8-8fee-756590eea613","resolution":{"observed_at":"2026-08-06T12:44:21.732658Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.736007Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":87,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.736007Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:214d3e2cbb9d62d09a5d335660bd2addb34afe31407aba9985bbad192c2c06f2","observation_id":"d30536c9-4172-4044-8786-5528383e4f5b","resolution":{"observed_at":"2026-08-06T12:44:21.736007Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.16789","last_updated":"2023-10-03T14:45:48Z","snapshot_observed_at":"2026-07-06T16:00:46.542753Z","submitted_at":"2023-07-31T15:56:53Z","title":"ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.16789","snapshot_observed_at":"2026-08-06T12:44:21.738675Z","title":"doi:10.48550/arXiv.2307.16789","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":88,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.738675Z"},"links":{"cited_paper":"/paper/2307.16789","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:1fcc301a27ce4329c459d5e16ce4a9eebc3d9d86b604b91ed85ecde9284f1488","observation_id":"b42bbf4c-8db5-45a2-910d-5e0cbcef3c63","resolution":{"observed_at":"2026-08-06T12:44:21.738675Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.03314","last_updated":"2023-06-05T23:55:37Z","snapshot_observed_at":"2026-08-11T04:53:44.472967Z","submitted_at":"2023-06-05T23:55:37Z","title":"Multi-Agent Collaboration: Harnessing the Power of Intelligent LLM Agents","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.03314","snapshot_observed_at":"2026-08-06T12:44:21.782168Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":89,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.782168Z"},"links":{"cited_paper":"/paper/2306.03314","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:a06ea85b5863de9ba7565b5ec500c1d3e752e2698907a655e1885de0442a2917","observation_id":"3df9fabe-2935-42bb-bc9e-917387192ed9","resolution":{"observed_at":"2026-08-06T12:44:21.782168Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.785167Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":90,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.785167Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:445d12fad2b146a6848b3e211fb11bd9d2c2a9a58c7601aeab51d9a06f312568","observation_id":"78653e92-0386-4bef-a808-53abdf6e81e3","resolution":{"observed_at":"2026-08-06T12:44:21.785167Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.747431Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":91,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.747431Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:cd1ea147f29cd7d5b50106213fd1d3a76ef1ba1c00ee87dd7742d867611e37a5","observation_id":"9e607f34-0109-49b2-9994-8cd5f0bcc300","resolution":{"observed_at":"2026-08-06T12:44:21.747431Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.08184","last_updated":"2024-06-12T13:14:50Z","snapshot_observed_at":"2026-08-13T21:10:47.214747Z","submitted_at":"2024-06-12T13:14:50Z","title":"MobileAgentBench: An Efficient and User-Friendly Benchmark for Mobile LLM Agents","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.08184","snapshot_observed_at":"2026-08-06T12:44:21.793466Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":92,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.793466Z"},"links":{"cited_paper":"/paper/2406.08184","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:4ab09259a343b7274300a3d331b3ad4ca0b33381faa03dc91a4530472fcb2920","observation_id":"e8dab5f9-2e2a-445e-8ec8-310bff789f07","resolution":{"observed_at":"2026-08-06T12:44:21.793466Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1145/3570945","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"Reimann, Catharine Oertel, Florian A","venue":null,"work_id":"b7adc62d-2fcb-4593-b6cc-e7f96a074613","year":2023},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":93,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.752880Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:1ee82de9627ce5308b25f80335aed64b527678181ca4da434e428306cd06111f","observation_id":"a939c8f0-8b53-4922-aa76-00d617573ea3","resolution":{"observed_at":"2026-08-06T12:44:22.059811Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T12:44:21.755622Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":94,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.755622Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:e0990b33fdc749215d2f4571dae7df7b861f776ad0257de3e5980cec564653f0","observation_id":"ff967955-d427-497b-a773-035c266ccb71","resolution":{"observed_at":"2026-08-06T12:44:21.755622Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.15817","last_updated":"2024-05-17T17:17:45Z","snapshot_observed_at":"2026-07-06T16:24:30.545494Z","submitted_at":"2023-09-25T17:08:02Z","title":"Identifying the Risks of LM Agents with an LM-Emulated Sandbox","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.15817","snapshot_observed_at":"2026-08-06T12:44:21.759303Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":95,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.759303Z"},"links":{"cited_paper":"/paper/2309.15817","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:cdd961bfcc42c46ff127016b639da39fb96001668788bd69a35e50879dc189fb","observation_id":"78cc4a51-1f84-4584-8b9f-0ba4a884a3f5","resolution":{"observed_at":"2026-08-06T12:44:21.759303Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.18760","last_updated":"2024-11-01T14:37:37Z","snapshot_observed_at":"2026-08-13T05:13:03.271734Z","submitted_at":"2023-11-30T18:02:44Z","title":"TaskBench: Benchmarking Large Language Models for Task Automation","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.18760","snapshot_observed_at":"2026-08-06T12:44:21.762114Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":96,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.762114Z"},"links":{"cited_paper":"/paper/2311.18760","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:0d05469363f752db7e5eaba9d441b2f8c74b6d678a13f874ea9f680582455f2a","observation_id":"8fd32775-a877-4f2f-8338-4d5d8e800dc2","resolution":{"observed_at":"2026-08-06T12:44:21.762114Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.05349","last_updated":"2024-11-08T06:12:56Z","snapshot_observed_at":"2026-08-12T22:05:03.414639Z","submitted_at":"2024-11-08T06:12:56Z","title":"Enhancing Cluster Resilience: LLM-agent Based Autonomous Intelligent Cluster Diagnosis System and Evaluation Framework","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.05349","snapshot_observed_at":"2026-08-06T12:44:21.765199Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":97,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.765199Z"},"links":{"cited_paper":"/paper/2411.05349","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:0d6755ef06b41709fdfea4966a1df09bb71cedc3bc1446dda9416cf01a879ee6","observation_id":"ab70d92b-935f-40c1-8002-2c449028ec71","resolution":{"observed_at":"2026-08-06T12:44:21.765199Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2022.findings-emnlp.359","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":null,"venue":null,"work_id":"a0ebdd2b-6e58-46d2-af8f-86a75d6ea4c4","year":2022},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":98,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.810777Z"},"links":{"citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:97e9aa037cd6d6e1d89e786d410b72a8adfc4e042011006857ed5c81fe616a48","observation_id":"85876bdb-f874-4a30-8cf2-1ece2333c195","resolution":{"observed_at":"2026-08-06T12:44:21.999884Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2311.08562","last_updated":"2024-11-27T12:25:28Z","snapshot_observed_at":"2026-08-13T05:25:13.805888Z","submitted_at":"2023-11-14T21:46:27Z","title":"MAgIC: Investigation of Large Language Model Powered Multi-Agent in Cognition, Adaptability, Rationality and Collaboration","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.08562","snapshot_observed_at":"2026-08-06T12:44:21.813422Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":99,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.813422Z"},"links":{"cited_paper":"/paper/2311.08562","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:134d75718dc951a101fa5cbcef0ade204624be5d433433375d4b2352d8691ee9","observation_id":"474df2fa-d0e8-460b-b8bf-1b720f13b87d","resolution":{"observed_at":"2026-08-06T12:44:21.813422Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.11363","last_updated":"2026-06-22T22:36:12Z","snapshot_observed_at":"2026-08-13T19:05:15.070072Z","submitted_at":"2024-09-17T17:13:19Z","title":"CORE-Bench: Fostering the Credibility of Published Research Through a Computational Reproducibility Agent Benchmark","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.11363","snapshot_observed_at":"2026-08-06T12:44:21.773326Z","title":"Siegel, Sayash Kapoor, Nitya Nagdir, Benedikt Stroebl, and Arvind Narayanan","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey","version":1},"reference_index":100,"source":"pdf_text","source_observed_at":"2026-08-06T12:44:21.773326Z"},"links":{"cited_paper":"/paper/2409.11363","citing_paper":"/paper/2507.21504"},"observation_digest":"sha256:44057618ace2ea08e3a1fc93f262baa0c84aa057f22a9372e9af6a627237a83d","observation_id":"13f5ad70-d8d5-4c0b-9765-0743eb507494","resolution":{"observed_at":"2026-08-06T12:44:21.773326Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2507.21504","last_updated":"2025-07-29T04:57:02Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-13T15:56:19.697323Z","submitted_at":"2025-07-29T04:57:02Z","title":"Evaluation and Benchmarking of LLM Agents: A Survey"},"reference_resolution":{"displayed":100,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":95,"verified_exact":5,"verified_fuzzy":0},"total_outbound_references":143},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"thesis":"As of 14 August 2026, this Paper Citation Record lists 100 of 143 outbound references and 6 inbound Pith citation observations for arXiv:2507.21504."}