{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:6C7HCQIM2M2U3YXEW3IH37SY3P","short_pith_number":"pith:6C7HCQIM","canonical_record":{"source":{"id":"2406.12321","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.AI","submitted_at":"2024-06-18T06:43:46Z","cross_cats_sorted":["cs.CL","cs.CV"],"title_canon_sha256":"3b8a919839c03b03384b5004e7e6b9afc7947791d6f20de8f962cda2aff614c1","abstract_canon_sha256":"d14dd07355dbf29fce827971d20ae5fb800334202f01151b8d2c4a31952f37e3"},"schema_version":"1.0"},"canonical_sha256":"f0be71410cd3354de2e4b6d07dfe58dbc8d35d335f69eb6440a660725814be35","source":{"kind":"arxiv","id":"2406.12321","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.12321","created_at":"2026-07-05T08:33:44Z"},{"alias_kind":"arxiv_version","alias_value":"2406.12321v1","created_at":"2026-07-05T08:33:44Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.12321","created_at":"2026-07-05T08:33:44Z"},{"alias_kind":"pith_short_12","alias_value":"6C7HCQIM2M2U","created_at":"2026-07-05T08:33:44Z"},{"alias_kind":"pith_short_16","alias_value":"6C7HCQIM2M2U3YXE","created_at":"2026-07-05T08:33:44Z"},{"alias_kind":"pith_short_8","alias_value":"6C7HCQIM","created_at":"2026-07-05T08:33:44Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:6C7HCQIM2M2U3YXEW3IH37SY3P","target":"record","payload":{"canonical_record":{"source":{"id":"2406.12321","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.AI","submitted_at":"2024-06-18T06:43:46Z","cross_cats_sorted":["cs.CL","cs.CV"],"title_canon_sha256":"3b8a919839c03b03384b5004e7e6b9afc7947791d6f20de8f962cda2aff614c1","abstract_canon_sha256":"d14dd07355dbf29fce827971d20ae5fb800334202f01151b8d2c4a31952f37e3"},"schema_version":"1.0"},"canonical_sha256":"f0be71410cd3354de2e4b6d07dfe58dbc8d35d335f69eb6440a660725814be35","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:33:44.444028Z","signature_b64":"9JQ8eZnPPL/+bbYB77JlvgJAy/gAqz8eUQbWQ6FoYT8qFWzloJLKiWmIQPswMJ2SPoeGBboUz2o/7w7PYzzACQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f0be71410cd3354de2e4b6d07dfe58dbc8d35d335f69eb6440a660725814be35","last_reissued_at":"2026-07-05T08:33:44.443655Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:33:44.443655Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.12321","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:33:44Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"FaFqEy+aMk3FDum1KcrIJgGIuJYtSEwxj0wA3A+EkD5gzpl7N8rJc2i5wygQR0Z9tv4qMay8x0gAY6uGDJxmAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-15T22:36:42.724457Z"},"content_sha256":"d99fa06d2c4e1954bea558b3008b443495c8a82b48d8df04cb9345471b60add5","schema_version":"1.0","event_id":"sha256:d99fa06d2c4e1954bea558b3008b443495c8a82b48d8df04cb9345471b60add5"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:6C7HCQIM2M2U3YXEW3IH37SY3P","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Automatic benchmarking of large multimodal models via iterative experiment programming","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.CL","cs.CV"],"primary_cat":"cs.AI","authors_text":"Alessandro Conti, Elisa Ricci, Enrico Fini, Massimiliano Mancini, Paolo Rota, Yiming Wang","submitted_at":"2024-06-18T06:43:46Z","abstract_excerpt":"Assessing the capabilities of large multimodal models (LMMs) often requires the creation of ad-hoc evaluations. Currently, building new benchmarks requires tremendous amounts of manual work for each specific analysis. This makes the evaluation process tedious and costly. In this paper, we present APEx, Automatic Programming of Experiments, the first framework for automatic benchmarking of LMMs. Given a research question expressed in natural language, APEx leverages a large language model (LLM) and a library of pre-specified tools to generate a set of experiments for the model at hand, and prog"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.12321","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.12321/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:33:44Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"302mWs6HPxvq7yz868u1gqRSmMgpemwtxRHz9coFYb/GlfCU2/Vzu0SQxDOkMp/LbSMdsywvZq59oQCE0RlRBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-15T22:36:42.725023Z"},"content_sha256":"439ae19999fb9889e1a3f6d721dbbe6e69d1383d41174c25b286213ce977c1d6","schema_version":"1.0","event_id":"sha256:439ae19999fb9889e1a3f6d721dbbe6e69d1383d41174c25b286213ce977c1d6"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/6C7HCQIM2M2U3YXEW3IH37SY3P/bundle.json","state_url":"https://pith.science/pith/6C7HCQIM2M2U3YXEW3IH37SY3P/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/6C7HCQIM2M2U3YXEW3IH37SY3P/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-15T22:36:42Z","links":{"resolver":"https://pith.science/pith/6C7HCQIM2M2U3YXEW3IH37SY3P","bundle":"https://pith.science/pith/6C7HCQIM2M2U3YXEW3IH37SY3P/bundle.json","state":"https://pith.science/pith/6C7HCQIM2M2U3YXEW3IH37SY3P/state.json","well_known_bundle":"https://pith.science/.well-known/pith/6C7HCQIM2M2U3YXEW3IH37SY3P/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:6C7HCQIM2M2U3YXEW3IH37SY3P","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"d14dd07355dbf29fce827971d20ae5fb800334202f01151b8d2c4a31952f37e3","cross_cats_sorted":["cs.CL","cs.CV"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.AI","submitted_at":"2024-06-18T06:43:46Z","title_canon_sha256":"3b8a919839c03b03384b5004e7e6b9afc7947791d6f20de8f962cda2aff614c1"},"schema_version":"1.0","source":{"id":"2406.12321","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.12321","created_at":"2026-07-05T08:33:44Z"},{"alias_kind":"arxiv_version","alias_value":"2406.12321v1","created_at":"2026-07-05T08:33:44Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.12321","created_at":"2026-07-05T08:33:44Z"},{"alias_kind":"pith_short_12","alias_value":"6C7HCQIM2M2U","created_at":"2026-07-05T08:33:44Z"},{"alias_kind":"pith_short_16","alias_value":"6C7HCQIM2M2U3YXE","created_at":"2026-07-05T08:33:44Z"},{"alias_kind":"pith_short_8","alias_value":"6C7HCQIM","created_at":"2026-07-05T08:33:44Z"}],"graph_snapshots":[{"event_id":"sha256:439ae19999fb9889e1a3f6d721dbbe6e69d1383d41174c25b286213ce977c1d6","target":"graph","created_at":"2026-07-05T08:33:44Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.12321/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Assessing the capabilities of large multimodal models (LMMs) often requires the creation of ad-hoc evaluations. Currently, building new benchmarks requires tremendous amounts of manual work for each specific analysis. This makes the evaluation process tedious and costly. In this paper, we present APEx, Automatic Programming of Experiments, the first framework for automatic benchmarking of LMMs. Given a research question expressed in natural language, APEx leverages a large language model (LLM) and a library of pre-specified tools to generate a set of experiments for the model at hand, and prog","authors_text":"Alessandro Conti, Elisa Ricci, Enrico Fini, Massimiliano Mancini, Paolo Rota, Yiming Wang","cross_cats":["cs.CL","cs.CV"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.AI","submitted_at":"2024-06-18T06:43:46Z","title":"Automatic benchmarking of large multimodal models via iterative experiment programming"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.12321","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d99fa06d2c4e1954bea558b3008b443495c8a82b48d8df04cb9345471b60add5","target":"record","created_at":"2026-07-05T08:33:44Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"d14dd07355dbf29fce827971d20ae5fb800334202f01151b8d2c4a31952f37e3","cross_cats_sorted":["cs.CL","cs.CV"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.AI","submitted_at":"2024-06-18T06:43:46Z","title_canon_sha256":"3b8a919839c03b03384b5004e7e6b9afc7947791d6f20de8f962cda2aff614c1"},"schema_version":"1.0","source":{"id":"2406.12321","kind":"arxiv","version":1}},"canonical_sha256":"f0be71410cd3354de2e4b6d07dfe58dbc8d35d335f69eb6440a660725814be35","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f0be71410cd3354de2e4b6d07dfe58dbc8d35d335f69eb6440a660725814be35","first_computed_at":"2026-07-05T08:33:44.443655Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:33:44.443655Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"9JQ8eZnPPL/+bbYB77JlvgJAy/gAqz8eUQbWQ6FoYT8qFWzloJLKiWmIQPswMJ2SPoeGBboUz2o/7w7PYzzACQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:33:44.444028Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.12321","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d99fa06d2c4e1954bea558b3008b443495c8a82b48d8df04cb9345471b60add5","sha256:439ae19999fb9889e1a3f6d721dbbe6e69d1383d41174c25b286213ce977c1d6"],"state_sha256":"67fde12ac982f6dcb3df4728cd900fae4bbec26e754f0c93acd09a2c57ac69aa"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Fc4jlRMWzlDjlyqQ0DFy9lSvH2vVZnqz1pSaljKVueZ4icObHlby/trdIElj/4DI+OAZV7ENmRyTGAxDAeXXDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-15T22:36:42.729203Z","bundle_sha256":"64f942672371e0b1847ea40d80024b0c993922c7b273e5f5f220ca44f19663e6"}}