{"as_of":"2026-08-09T20:10:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:749280c22ed973ce18f671035288e9ca45b2f7465a600f89f1849088b6ac0eb2","coverage":[{"denominator":49,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":49,"source":"paper_references, paper_reference_links","source_observed_at":"2026-07-14T13:17:55.700527Z","state":"measured"},{"denominator":49,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":49,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-09T06:31:02.800959+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2607.10240/citation-record","integrity":"/paper/2607.10240/integrity","json":"/paper/2607.10240/citation-record.json","paper":"/paper/2607.10240"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:2ed7b5af15d985ede45574654d6a232e574fdf678706ba3436f59275c50ad6d6","observation_id":"26b102c3-a43b-4ec0-8008-0eedf637e57a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"Lawrence Zitnick, and Devi Parikh","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:474e503256bc64327ba9478e4694f3c60f644a61fd19249cc6d35b6150714b88","observation_id":"2a441827-94cb-4111-8217-e72426d2217a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"InProceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:e6f28be882ee978941a0c8f66079a6a9c7d7c7931e54ddce7715630d7c78fd24","observation_id":"a8fe680c-579b-4ba4-9c6c-de4ee052b74d","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.13923","last_updated":"2025-02-19T18:00:14Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-02-19T18:00:14Z","title":"Qwen2.5-VL Technical Report","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.13923","snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"cited_paper":"/paper/2502.13923","citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:7a1ff4cc622b4a008ecdc17817e4eba6f2b33f47bed13a5ef4ec5a73621decfc","observation_id":"174b0ce4-7f06-4366-a379-ea746e09bf3c","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:fdedf4a2b4369105b18a568f0b6cc9ca4d89b0a49f44a1bce44b864c0a265087","observation_id":"978781ae-ec38-4684-8736-ef0e8b22037e","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:3e50e43a50a87fecb3e43960f65db33519d5c0a553e956070096f36cac8c7438","observation_id":"cead67fd-ee82-42c8-b0e2-61de3fb884f2","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:5b28e5ef78cf5a73627c1212e368a7105bcc05e6e1e54765f63d82135ef7c545","observation_id":"1158f281-8989-4d84-8274-fa8c3a7ed59a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:8c5c468f2a22302faf222610f39bf03eba37388a6abbf8dd0010f7e46a82ed92","observation_id":"1dd1c501-ba03-440e-b893-ced9f776338a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"In Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:9859438e63865f7a50ff8991dfed43bb14cd0a7083afcbcb9b694f53fac9e8f5","observation_id":"42020f16-b031-47c0-929b-3d1dd330a494","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:f2bbbadcc2f15924f944045ceaaaeb3c76d78893443336afcea88fdadb2d1787","observation_id":"af2a2ebd-ed25-44e5-9521-cf5b8abcbb19","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.19786","last_updated":"2025-03-25T15:52:34Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-03-25T15:52:34Z","title":"Gemma 3 Technical Report","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.19786","snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"cited_paper":"/paper/2503.19786","citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:99cd8f7a4d28b7c9d3c0bd96c4b0d84abbf7a45f00ba11fe3b443e19dfe0bef8","observation_id":"5793357f-1a46-499b-baf2-d75f22d36c3f","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.07270","last_updated":"2024-05-05T20:34:28Z","snapshot_observed_at":"2026-08-07T17:45:50.717841Z","submitted_at":"2024-02-11T18:26:18Z","title":"Open-ended VQA benchmarking of Vision-Language models by exploiting Classification datasets and their semantic hierarchy","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.07270","snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"Bravo, and Thomas Brox","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"cited_paper":"/paper/2402.07270","citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:b956d2c88f2afc91c03fcfceb1e5ce57b7a53fc8df3c99eab6093a2b8e004479","observation_id":"b0f029c4-825f-48b4-a21b-d377385af2f8","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21783","last_updated":"2024-11-23T23:27:33Z","snapshot_observed_at":"2026-07-06T18:55:11.576666Z","submitted_at":"2024-07-31T17:54:27Z","title":"The Llama 3 Herd of Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21783","snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"cited_paper":"/paper/2407.21783","citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:cda86fb26932a054c1d6a36a9eae35d26def9389969c9f2040c4aa851d3ac838","observation_id":"bf5de91a-d61c-46bf-b96d-82e8b784efdf","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:87ed53376a29a0f1163af5f751be02d86d82de9642bcde06c1f0402071d3f21d","observation_id":"1ca64184-b7b6-487a-bd76-4307e054e4e5","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:e862354b3cd2515e6e949bb7c2010bc4e0826ea41e87485641e3eb1204cb6001","observation_id":"797f9128-dc06-4d91-8294-3e45b0f70872","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"InFind- ings of the Association for Computational Linguistics: EMNLP","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:5c8b85e67eada472765d4a92bb5474e22505d7f26116f87462fc6cec676ec49d","observation_id":"8f6d4e81-69c7-4d8f-b77f-7f7249b814fc","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:bf086e4a2f00cb43ca43fec927e5d6eb0b92291b488f53977585f6033e922555","observation_id":"09c494c6-cb8f-44ad-ad6b-52d062b925d3","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:097ea15612d6bdd8e0e855030dcf69fdcd02407cdddad08d1a8e761a281b7b5a","observation_id":"ea7664fd-f28a-41ba-afc8-d2e8353bafab","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:c6f32ed0febdf2611575fca2421bd125d98b27d38bb41f5a7e3e79387b208bda","observation_id":"9f08e2be-a55b-4834-b447-9c027527f1e1","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:a5e8b30667e416d09e556ba522830600b1d7a94f0273cd4199cf1710aadb0d22","observation_id":"9819454d-ae5e-4800-b61d-6c7df4bda32f","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"Gonzalez, Hao Zhang, and Ion Stoica","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:4bd40ff710ed68819f11f8754e7075cc8fba81da5eeaa54c5b1a349096094310","observation_id":"3aecee85-bf54-4213-8cf2-59105151aa42","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:7aacebd0281c9f2a3ce55cf6989466e9c58471dd95020a7b7655b63ff964b5fd","observation_id":"c5df8434-33d6-46e0-843b-502f1b615588","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:a60ee298344a314f2f2811083a821d228e7d1b2156d33db1f63b3e95ea9494e2","observation_id":"05614de5-8417-40e8-bdae-395a180d34b2","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:8d1976fa4a1d7af9e12628266caaa7287435ac1c983abe7f546156d96310219f","observation_id":"eb0f4217-f738-4f4d-be83-00b99dc2619e","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:8cb4d0e473c6c3c2b9cf01072a23b847acf64bfbdd1a55bd2100efac2109e277","observation_id":"cd3131db-2d22-4da2-821a-034e5b8c8f5a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:d13e573dd29b0be7938cf88ed33852cb38a3f0ecfdf44e0da8ca0801d20f98c4","observation_id":"c62d589b-bdb4-43d0-bbee-1b4e4fc9d59a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:91d299ff008f2225bde3657bf2dec937a87badc21d296e23a12e15e3bf492d27","observation_id":"6086f200-4f9d-4d24-a3b4-46876332ce3e","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:797fc19ab6cec2f2a2ae5fb12dd80bdefa80b2ee54daacd9c2a455146dd3218c","observation_id":"0de7136f-b488-49f6-abbb-cd4c04b32719","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:a913edb245554bd76b7e07d70a80ab28327e8e22a59a3fd74f52caff025e4a5b","observation_id":"6ab9deb0-9a4b-4c98-b7cd-9c8155674bfe","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:d3212977558a1da21b700d24e7a2e5ee2797313b18f67903f2e45b8aaea94f8d","observation_id":"6d1722f6-70d4-4508-b0fd-39718cd21532","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:71094ebde862bcc24e0848efaa011dcd0b2584006b7cda32ae64b5df4fe2ca15","observation_id":"3401c0ef-217c-40bb-a986-7f33841017bf","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:c5affa459cbf4928c983211745bfbf613a6ad5656f96724e0e85e8b16574282c","observation_id":"868d5bfd-0ceb-44ce-8c61-866da95c08d4","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:6eeaff8efc98ff3b3f09641acea38400646b82ab3879fdd38b963486da585203","observation_id":"d2292abf-2aca-4673-ade5-5e44490ebcb8","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.03848","last_updated":"2025-04-22T14:11:50Z","snapshot_observed_at":"2026-08-06T01:28:29.570476Z","submitted_at":"2024-02-06T09:50:08Z","title":"ANLS* -- A Universal Document Processing Metric for Generative Large Language Models","version":10},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.03848","snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"cited_paper":"/paper/2402.03848","citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:36e9dbfa78988d0327e130c6ebc4d666cdf214483d3e54560aede431796b689c","observation_id":"733539bd-7344-42e5-bdfb-3d6a08b28efc","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:842250cee55296ce0db0adcd3dc263d6829f48fb882b8d58e1c4e27be7e3c893","observation_id":"8624e63b-b5ef-40bd-8832-895beea429ba","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:6ecbe7b26ee52fb8c47be92a74cc8a8e6e358145a89e38fdda1c4e6f342f4ade","observation_id":"8000e368-7a7b-4285-8c2a-1dd293331a3a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:71a992468023f35ee10e6086e5490ec71a4bcc957382a3a85df76220405ffd10","observation_id":"2a0b000b-fd5c-40cf-82c9-914acbfca9df","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"Le, Ed H","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:bc53a8a8cf7ae0f16b3f661feb4ed107f02bc8069b92444fbdebeaae9ac03563","observation_id":"2ec11b24-cda5-46bb-9021-32a9bbafc844","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"InProceedings of the International Conference on Learning Representations (ICLR)","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:365a3479bc6d29ec597d1872c62e733643999a81390fb6fbd9523cca3ec3429e","observation_id":"da78a49b-cfd7-4d62-b0f8-ee25765bdefb","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:3ac95831f02c96a6b772009c3b42fca94dec59c0e4756f4d77cb38b21e91eb3d","observation_id":"8c253bc3-e8ad-46cf-acb1-07b3173a5ed9","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:c212fde73a8725fc9eca60f4322aeccd51b4d3af5986a90c5b60ff67ed7ae397","observation_id":"cb3fcd5e-5456-40ce-8d82-dd65fe8e264d","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:ee2aff8b2c80d22ac2a592f8dc0c5d05dbbc6838afc0c107d4c9857ca91744f1","observation_id":"3d09161f-6065-4973-905d-f8c4b8e4e035","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"Weld, and Jonathan Berant","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:ded1da6873f93986910792125e86264279e11b1f01ab67c724d6c03954b2318d","observation_id":"2ec93f55-ccb3-43d6-90dd-c4e8e7199fd0","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:6eed070ce9d8026035d605c0c267b085c09f23a5bf51b2b6700d4976a061e37b","observation_id":"190bb2dc-22b6-4241-bf9e-166d1a6817ea","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2606.06865","last_updated":"2026-06-05T03:24:38Z","snapshot_observed_at":"2026-07-06T23:46:37.903443Z","submitted_at":"2026-06-05T03:24:38Z","title":"Are Large Language Models Suitable for Graph Computation? Progress and Prospects","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2606.06865","snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"cited_paper":"/paper/2606.06865","citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:a25b5e2f8999730da7efb8c1bd9c294bc482efe66fda18b2cd06f2a3482a2bcf","observation_id":"178c6799-d934-45f7-a11f-291314743a04","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"Xing, et al","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:1802a91e1c890f883dce271fcb4e69f4efe4d185d358acc1da8b3aaeda31d3a8","observation_id":"ee4cec9d-a0bb-4d81-978f-b47215c88469","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:84b63ac452a84af32caa398432186ca488b55d2743bf2255da764e5638e229ea","observation_id":"73a301a1-6b82-4671-9999-27aa813222c7","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:b9ae8fff4942d25fc513ce9d6a056b87f240ee29a3254a0be96dbebdc4c323a8","observation_id":"5bed46e7-fdf3-4f39-a509-fc3466cd52fc","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2504.10479","last_updated":"2025-04-19T03:47:21Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-04-14T17:59:25Z","title":"InternVL3: Exploring Advanced Training and Test-Time Recipes for Open-Source Multimodal Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2504.10479","snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"official reversal","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"cited_paper":"/paper/2504.10479","citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:3d60483bb1408be15e89ba992df34f2a674054341d60d5c588df9264199bc915","observation_id":"c177fff9-953b-4a7a-a737-af2d8a944b0a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","latest_version":1,"primary_category":"cs.CV","snapshot_observed_at":"2026-08-09T07:41:09.133389Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks"},"reference_resolution":{"displayed":49,"state_counts":{"malformed_identifier":2,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":47,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":49},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"thesis":"As of 9 August 2026, this Paper Citation Record lists 49 of 49 outbound references and 0 inbound Pith citation observations for arXiv:2607.10240."}