{"as_of":"2026-08-10T20:19:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:ae25b7f3b0e22d7a078a148f659a422daebeccebbc89f36923661bd3bb4dd4c8","coverage":[{"denominator":28,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":28,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T05:20:15.085691Z","state":"measured"},{"denominator":28,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":28,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-10T06:31:04.303077+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2608.06329/citation-record","integrity":"/paper/2608.06329/integrity","json":"/paper/2608.06329/citation-record.json","paper":"/paper/2608.06329"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:13.564295Z","title":"2024 , eprint=","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:13.564295Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:dd1f98cc807142420c77cf854058f96dc6ff46b0f644745bd41191e2a34f146b","observation_id":"6e8c73ef-a157-47f0-be59-8223d60498e9","resolution":{"observed_at":"2026-08-07T05:20:13.564295Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.821241Z","title":"Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing , pages=","venue":null,"work_id":"0c9dbc30-de3a-445e-acc0-2498ba0fb5ba","year":2024},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:13.617659Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:1e44c30c423dc641e956aa1c20488cc4af7ddcf54ee0733b2df5161f798a6f40","observation_id":"4321654b-8ad3-42f8-95c0-f28b5362a22f","resolution":{"observed_at":"2026-08-07T05:20:15.827288Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.804470Z","title":"Biometrika , volume =","venue":null,"work_id":"5dfb8d2b-e3e9-48d1-b12e-10dbee39aa03","year":1945},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:13.694182Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:3e970b86cbfce436e845463c6b5fa01abd6f71a5b6a2ff2c7ce1d643f43b06a0","observation_id":"8040c8a6-c499-417a-8c05-6840c8e3878e","resolution":{"observed_at":"2026-08-07T05:20:15.809376Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:13.736829Z","title":"2025 , eprint=","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:13.736829Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:ca1eac89e91077f9e8d6d052ff5743f22a23288bb80dd58b5066d9cb3cf3c1ec","observation_id":"33a569df-e775-487f-bb0a-c97a2bdc4d03","resolution":{"observed_at":"2026-08-07T05:20:13.736829Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.772757Z","title":"2025 , eprint=","venue":null,"work_id":"9f9d2d7c-380c-42ae-a733-f15b8160cee1","year":2025},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:13.819650Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:031fd43293b54764c553f97f8bb37d9ce8efe3ed982d4756a46c065c32547bac","observation_id":"e04b2ca1-7090-40a4-a415-791ff2786571","resolution":{"observed_at":"2026-08-07T05:20:15.777958Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.745302Z","title":null,"venue":null,"work_id":"b2b9efa4-92ba-411c-88db-61e1a17b5d62","year":null},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:13.897906Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:f7ac6e207a64dad50a6c2a4e73ef60a3bfb59ccd8a12d9433bc17555bd03798b","observation_id":"cb609a9d-396d-4b08-853a-658f33a21848","resolution":{"observed_at":"2026-08-07T05:20:15.753484Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.721758Z","title":"2024 , eprint=","venue":null,"work_id":"82c37c8a-3330-49f9-90b6-2981f56666b1","year":2024},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:13.967434Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:64b9c357b9800857b67dbdadd035a66b7dee09f9efe4609d50a2f5fc6de1fdad","observation_id":"f0579000-f432-48ad-905b-a26e96170a52","resolution":{"observed_at":"2026-08-07T05:20:15.729328Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:14.026399Z","title":"M ulti WOZ - A Large-Scale Multi-Domain W izard-of- O z Dataset for Task-Oriented Dialogue Modelling","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:14.026399Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:5527d1e3deb3755d9cc82934af9f00e9e63a63604af80bc3198a3cd542b4458b","observation_id":"923874bb-2a78-4d38-b47c-ee4ca853ffa1","resolution":{"observed_at":"2026-08-07T05:20:14.026399Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:14.089691Z","title":"Towards Enforcing Company Policy Adherence in Agentic Workflows","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:14.089691Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:26a952b2cf5cff2516679979742d40f63fbc0bb556bd0de829772bb0113450ac","observation_id":"3a990bcc-b74d-43c0-abf1-1d4e00836877","resolution":{"observed_at":"2026-08-07T05:20:14.089691Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:14.151335Z","title":"2025 , isbn =","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:14.151335Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:57bb422d07fd3f323bd9e1590ce932a4e466a1f3e349e94116a26f6269d0e9fe","observation_id":"ec4e96bf-2dd3-4f30-8b55-83a75ea8116a","resolution":{"observed_at":"2026-08-07T05:20:14.151335Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.694678Z","title":"2026 , eprint=","venue":null,"work_id":"ba6fe18b-9f1f-4ea4-a3bb-9618cc3ce05f","year":2026},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:14.233010Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:39e452246c831eaa4393dd45409df577110f3fb06289950eb75d6ca1156305bf","observation_id":"07bee70b-4dee-4424-a1eb-4815677def87","resolution":{"observed_at":"2026-08-07T05:20:15.709399Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.676331Z","title":"2024 , howpublished=","venue":null,"work_id":"76ce5149-49ef-4585-9142-e2ffad32dc77","year":2024},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:14.289143Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:173c3a854d0dfc7a144c10505faf4b7135eaa7302968464a3216f55846121af9","observation_id":"5aed78d5-ae0e-4233-8735-edbc776c98e4","resolution":{"observed_at":"2026-08-07T05:20:15.682363Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:14.353384Z","title":"Advances in Neural Information Processing Systems , volume=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:14.353384Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:f5c0f7b5c2cbcbe84759e69a221bd5c55a8503c3e2d2dd66ea00e910426ff307","observation_id":"a73f6982-9722-48bd-9a5c-5a2056b43704","resolution":{"observed_at":"2026-08-07T05:20:14.353384Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:14.410620Z","title":"Advances in Neural Information Processing Systems , volume=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:14.410620Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:751d5688343d20157eaa0b6eb5dfdf7dde212c8cc8022f477f2c42be7434fdd0","observation_id":"9d011802-59c6-41d8-8f56-d1b6d9833a72","resolution":{"observed_at":"2026-08-07T05:20:14.410620Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.601521Z","title":"ArXiv , year=","venue":null,"work_id":"bb9b9884-f3f0-4f46-8145-53fc4288714b","year":null},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:14.467464Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:2efde07e8a54457d415a18f6b5658831af865239d2101377c65fc9ebabac5331","observation_id":"42d4dd64-0b58-4ab4-9b0c-87ddca914997","resolution":{"observed_at":"2026-08-07T05:20:15.608631Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2307.13854","last_updated":"2024-04-16T15:13:18Z","snapshot_observed_at":"2026-08-06T12:47:01.809185Z","submitted_at":"2023-07-25T22:59:32Z","title":"WebArena: A Realistic Web Environment for Building Autonomous Agents","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.13854","snapshot_observed_at":"2026-08-07T05:20:14.525116Z","title":"arXiv preprint arXiv:2307.13854 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:14.525116Z"},"links":{"cited_paper":"/paper/2307.13854","citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:5178b05f9b8804eaaa8d89e26e6829c2098e7d4c95c3dd8fce2c4bfc1f5a187c","observation_id":"65f94613-f33d-4790-b46e-859ed12cbb77","resolution":{"observed_at":"2026-08-07T05:20:14.525116Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:14.590129Z","title":"A pp W orld: A Controllable World of Apps and People for Benchmarking Interactive Coding Agents","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:14.590129Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:8d0f9578dd4516f8917ba94d8aa40ad780ea95151648d6aada0d60261a502efc","observation_id":"569c2335-0adc-4805-8838-fe1c4ae066e8","resolution":{"observed_at":"2026-08-07T05:20:14.590129Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.12115","last_updated":"2025-05-29T23:07:34Z","snapshot_observed_at":"2026-08-10T07:47:59.443590Z","submitted_at":"2025-02-17T18:41:16Z","title":"SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real-World Freelance Software Engineering?","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.12115","snapshot_observed_at":"2026-08-07T05:20:14.667222Z","title":"arXiv preprint arXiv:2502.12115 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:14.667222Z"},"links":{"cited_paper":"/paper/2502.12115","citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:af754e0fb4a856a8f61059913c2e7fcbc8f5cd17fdf21cf121048de527dfeeed","observation_id":"e8662d9f-a520-4e7d-a1b0-f9b044b80802","resolution":{"observed_at":"2026-08-07T05:20:14.667222Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.576216Z","title":"2024 , eprint=","venue":null,"work_id":"7bc037f5-8b66-4570-91ea-b81a3461cbb0","year":2024},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:14.747455Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:d3664c090868d84815ea29fda82e286233af09d9af34f753714d038377e9b80b","observation_id":"8b8a38a0-df2f-48e9-ab09-ec4151a41e89","resolution":{"observed_at":"2026-08-07T05:20:15.585487Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:14.813652Z","title":"and Zhang, Hao and Gonzalez, Joseph E","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:14.813652Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:044b970eb6a291bbd1f9a5e9ab447ff06cd068546ead72355567bddda77809e5","observation_id":"33013a20-0739-4f9d-a0da-e56a09d54e18","resolution":{"observed_at":"2026-08-07T05:20:14.813652Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.543655Z","title":null,"venue":null,"work_id":"b5eeecc1-f63c-4c50-922a-ed042044e90f","year":null},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:14.902068Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:997965fbfe9efa4e439aea6f6342bb0592c08a6e1d8bc3359c8019e0300fea16","observation_id":"a668dfd3-b58e-4b32-98a8-f179a111838f","resolution":{"observed_at":"2026-08-07T05:20:15.548321Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.524755Z","title":"2025 , eprint=","venue":null,"work_id":"bae6ba27-29e8-4c8c-ba7d-b14389e7eacf","year":2025},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:14.950594Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:f43a51509b73f32199015add2f8c1bce69c7a664b701585c015d215b6504ba26","observation_id":"fd45945e-cf58-48d6-b002-9f6436f95937","resolution":{"observed_at":"2026-08-07T05:20:15.529234Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.028255Z","title":"Proceedings of the 2023 conference on empirical methods in natural language processing , pages=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:15.028255Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:14ac81f7ebaf180e51ca8e9e457757172f771cd57c52bcb3d0db1dda2e64f802","observation_id":"3805642d-3eed-4869-a7c6-8420e0da7aea","resolution":{"observed_at":"2026-08-07T05:20:15.028255Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.062796Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:15.062796Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:4ed74cb22b92339f62dd8528b0b12c7f75696289421fb17ab93050b2afaf3b39","observation_id":"ec17f9f3-bc3a-4b86-ada7-1936ac1ac5e9","resolution":{"observed_at":"2026-08-07T05:20:15.062796Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21783","last_updated":"2024-11-23T23:27:33Z","snapshot_observed_at":"2026-08-10T16:40:37.411115Z","submitted_at":"2024-07-31T17:54:27Z","title":"The Llama 3 Herd of Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21783","snapshot_observed_at":"2026-08-07T05:20:15.067564Z","title":"arXiv preprint arXiv:2407.21783 , year=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:15.067564Z"},"links":{"cited_paper":"/paper/2407.21783","citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:9cf862796af1e90b3de2d36041725b0ea294b61cabba39af0f3b4dd295d21732","observation_id":"990bf10c-8a45-4186-a9b0-db48f839a5bc","resolution":{"observed_at":"2026-08-07T05:20:15.067564Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.455900Z","title":"Proceedings of the 41st International Conference on Machine Learning , articleno =","venue":null,"work_id":"327226e9-fb6b-4abd-940d-ee5e5d6f8da0","year":2024},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:15.073683Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:9689784c1011a752bd30cd864141744ba8d42fd413f796a7ead157c8d29975d0","observation_id":"aea02b38-81f3-4306-9d2c-e7f5699180e3","resolution":{"observed_at":"2026-08-07T05:20:15.468783Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-10T06:31:04.303077+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.079297Z","title":"2023 , eprint=","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:15.079297Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:01b1b89159b62c2b82065b099bbc757276fcf7a8d3c8de1fe403ad1663da6521","observation_id":"931892eb-34c1-4a52-a619-51171f68ccb5","resolution":{"observed_at":"2026-08-07T05:20:15.079297Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T05:20:15.085691Z","title":"Aligning Large Language Models through Synthetic Feedback","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-07T05:20:15.085691Z"},"links":{"citing_paper":"/paper/2608.06329"},"observation_digest":"sha256:1f4fee46a6747b699d2a058bea74b08ea821858de2c02977ff0be18a5204a577","observation_id":"5df0d76d-c106-4fb2-b600-e5bf2446b874","resolution":{"observed_at":"2026-08-07T05:20:15.085691Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2608.06329","last_updated":"2026-08-06T17:39:21Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-10T07:48:52.801303Z","submitted_at":"2026-08-06T17:39:21Z","title":"Benchmarking the Benchmarks: Evaluating Benchmarks for Conversational Agents"},"reference_resolution":{"displayed":28,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":18,"verified_exact":0,"verified_fuzzy":10},"total_outbound_references":28},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-10T06:31:04.303077+00:00","source":"crossref"},{"observed_at":"2026-08-10T06:30:57.382061+00:00","source":"retraction_watch"}],"thesis":"As of 10 August 2026, this Paper Citation Record lists 28 of 28 outbound references and 0 inbound Pith citation observations for arXiv:2608.06329."}