{"as_of":"2026-08-12T22:39:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:95718d45630cb2c388eb484001adee792c9a7817efcf7574fc6ea672ebab9551","coverage":[{"denominator":0,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":12,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":12,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-12T06:34:41.77262+00:00","state":"measured"},{"denominator":12,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":12,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T16:16:01.149922Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-05-20T19:13:40.348932Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2406.19470","last_updated":"2024-11-11T02:27:54Z","snapshot_observed_at":"2026-07-06T18:38:09.490968Z","submitted_at":"2024-06-27T18:21:32Z","title":"Changing Answer Order Can Decrease MMLU Accuracy","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.19470","snapshot_observed_at":"2026-08-12T16:16:01.149922Z","title":"Chang- ing answer order can decrease mmlu accuracy,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13757","last_updated":"2025-07-01T23:27:52Z","snapshot_observed_at":"2026-08-12T15:52:57.444740Z","submitted_at":"2024-11-21T00:01:51Z","title":"GenBFA: An Evolutionary Optimization Approach to Bit-Flip Attacks on LLMs","version":4},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-12T16:16:01.149922Z"},"links":{"cited_paper":"/paper/2406.19470","citing_paper":"/paper/2411.13757"},"observation_digest":"sha256:bd009dedfd3ff3477d7eeca1452d46cc6b23d8c33dd29ff7e5a6086cf3850c36","observation_id":"e02a791e-bf25-4157-9fa1-512b9e929dbd","resolution":{"observed_at":"2026-08-12T16:16:01.149922Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19470","last_updated":"2024-11-11T02:27:54Z","snapshot_observed_at":"2026-07-06T18:38:09.490968Z","submitted_at":"2024-06-27T18:21:32Z","title":"Changing Answer Order Can Decrease MMLU Accuracy","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.19470","snapshot_observed_at":"2026-08-11T21:37:26.940619Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.04277","last_updated":"2024-12-05T15:59:29Z","snapshot_observed_at":"2026-08-12T00:48:46.047402Z","submitted_at":"2024-12-05T15:59:29Z","title":"Arabic Stable LM: Adapting Stable LM 2 1.6B to Arabic","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-11T21:37:26.940619Z"},"links":{"cited_paper":"/paper/2406.19470","citing_paper":"/paper/2412.04277"},"observation_digest":"sha256:7cb798d280cba5d79eb077813af8dcc08fdff26d5578f20477f83b2f9a890b45","observation_id":"2857396a-2651-45c1-8a74-0d9036d5753b","resolution":{"observed_at":"2026-08-11T21:37:26.940619Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19470","last_updated":"2024-11-11T02:27:54Z","snapshot_observed_at":"2026-07-06T18:38:09.490968Z","submitted_at":"2024-06-27T18:21:32Z","title":"Changing Answer Order Can Decrease MMLU Accuracy","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.19470","snapshot_observed_at":"2026-08-11T17:34:19.662284Z","title":"Changing answer order can decrease mmlu accuracy, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.08819","last_updated":"2024-12-11T23:31:06Z","snapshot_observed_at":"2026-08-11T21:32:41.724258Z","submitted_at":"2024-12-11T23:31:06Z","title":"HARP: A challenging human-annotated math reasoning benchmark","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-11T17:34:19.662284Z"},"links":{"cited_paper":"/paper/2406.19470","citing_paper":"/paper/2412.08819"},"observation_digest":"sha256:f618f1e7195978bca9f0f7db5836e06c63b3b5a5f229e09823c8297408fb1b56","observation_id":"90f8dd49-73ab-4ce4-93e8-3b5545222d3d","resolution":{"observed_at":"2026-08-11T17:34:19.662284Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19470","last_updated":"2024-11-11T02:27:54Z","snapshot_observed_at":"2026-07-06T18:38:09.490968Z","submitted_at":"2024-06-27T18:21:32Z","title":"Changing Answer Order Can Decrease MMLU Accuracy","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.19470","snapshot_observed_at":"2026-08-11T15:55:23.169547Z","title":"Changing answer order can decrease mmlu accuracy, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.10558","last_updated":"2024-12-13T21:03:10Z","snapshot_observed_at":"2026-08-12T14:47:39.822958Z","submitted_at":"2024-12-13T21:03:10Z","title":"Too Big to Fool: Resisting Deception in Language Models","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-11T15:55:23.169547Z"},"links":{"cited_paper":"/paper/2406.19470","citing_paper":"/paper/2412.10558"},"observation_digest":"sha256:878fdc909d4759486e68549145f837e4c4780385cc50f5ce3ba6928219010f9d","observation_id":"b30f42db-93d8-4bae-9d56-2409d23f0d8f","resolution":{"observed_at":"2026-08-11T15:55:23.169547Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19470","last_updated":"2024-11-11T02:27:54Z","snapshot_observed_at":"2026-07-06T18:38:09.490968Z","submitted_at":"2024-06-27T18:21:32Z","title":"Changing Answer Order Can Decrease MMLU Accuracy","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.19470","snapshot_observed_at":"2026-08-11T11:36:56.261554Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.15194","last_updated":"2024-12-19T18:58:04Z","snapshot_observed_at":"2026-08-12T04:44:06.445319Z","submitted_at":"2024-12-19T18:58:04Z","title":"MMLU-CF: A Contamination-free Multi-task Language Understanding Benchmark","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-11T11:36:56.261554Z"},"links":{"cited_paper":"/paper/2406.19470","citing_paper":"/paper/2412.15194"},"observation_digest":"sha256:515e2ffec6c6553a4722dbb5a437f511dd4c3088051375618d9117d10696cb10","observation_id":"a19bbbb6-d29e-45bf-bdab-8992c9615c43","resolution":{"observed_at":"2026-08-11T11:36:56.261554Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19470","last_updated":"2024-11-11T02:27:54Z","snapshot_observed_at":"2026-07-06T18:38:09.490968Z","submitted_at":"2024-06-27T18:21:32Z","title":"Changing Answer Order Can Decrease MMLU Accuracy","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.19470","snapshot_observed_at":"2026-08-08T16:11:43.132926Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.08662","last_updated":"2025-06-02T04:54:00Z","snapshot_observed_at":"2026-08-09T07:57:07.000139Z","submitted_at":"2025-02-10T09:34:15Z","title":"RoToR: Towards More Reliable Responses for Order-Invariant Inputs","version":3},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-08T16:11:43.132926Z"},"links":{"cited_paper":"/paper/2406.19470","citing_paper":"/paper/2502.08662"},"observation_digest":"sha256:461901d7c4e03cb669e42576d70ff7ca4ed9fb5cb922fb2712f8a14619163db6","observation_id":"cdbfaff9-978e-4087-96fd-5f1321e991a6","resolution":{"observed_at":"2026-08-08T16:11:43.132926Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19470","last_updated":"2024-11-11T02:27:54Z","snapshot_observed_at":"2026-07-06T18:38:09.490968Z","submitted_at":"2024-06-27T18:21:32Z","title":"Changing Answer Order Can Decrease MMLU Accuracy","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.19470","snapshot_observed_at":"2026-08-06T23:20:18.169890Z","title":"Gupta, D","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18710","last_updated":"2025-07-01T15:49:58Z","snapshot_observed_at":"2026-08-10T17:04:52.375523Z","submitted_at":"2025-06-23T14:49:01Z","title":"Benchmarking the Pedagogical Knowledge of Large Language Models","version":3},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-06T23:20:18.169890Z"},"links":{"cited_paper":"/paper/2406.19470","citing_paper":"/paper/2506.18710"},"observation_digest":"sha256:0c3f64f82976d2796ac2298a4c510553b63e10be4ac75c49f80e1f8a16a17038","observation_id":"b6d1f93e-e3dd-4009-9000-70e8e9a4973f","resolution":{"observed_at":"2026-08-06T23:20:18.169890Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19470","last_updated":"2024-11-11T02:27:54Z","snapshot_observed_at":"2026-07-06T18:38:09.490968Z","submitted_at":"2024-06-27T18:21:32Z","title":"Changing Answer Order Can Decrease MMLU Accuracy","version":2},"cited_work":{"arxiv_id":"2406.19470","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2406.19470","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Changing answer order can decrease MMLU accuracy.arXiv preprint arXiv:2406.19470","venue":null,"work_id":"9464f44f-c7eb-4b90-85b3-311c3aa8dc69","year":2024},"citing_paper":{"arxiv_id":"2507.23009","last_updated":"2026-05-11T11:47:15Z","snapshot_observed_at":"2026-07-06T22:05:22.373440Z","submitted_at":"2025-07-30T18:14:35Z","title":"Position: Stop Evaluating AI with Human Tests, Develop Principled, AI-specific Tests instead","version":2},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-05-19T02:12:48.586913Z"},"links":{"cited_paper":"/paper/2406.19470","citing_paper":"/paper/2507.23009"},"observation_digest":"sha256:b450389387dee15dc94645aaaa1ded205bd07f9e867a9a1c2a4c873dff6269eb","observation_id":"52594fd5-0e98-41bc-a90e-0ea4c98c651b","resolution":{"observed_at":"2026-05-19T02:12:55.777888Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19470","last_updated":"2024-11-11T02:27:54Z","snapshot_observed_at":"2026-07-06T18:38:09.490968Z","submitted_at":"2024-06-27T18:21:32Z","title":"Changing Answer Order Can Decrease MMLU Accuracy","version":2},"cited_work":{"arxiv_id":"2406.19470","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2406.19470","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Changing answer order can decrease MMLU accuracy.arXiv preprint arXiv:2406.19470","venue":null,"work_id":"9464f44f-c7eb-4b90-85b3-311c3aa8dc69","year":2024},"citing_paper":{"arxiv_id":"2603.09678","last_updated":"2026-05-11T23:17:28Z","snapshot_observed_at":"2026-07-06T22:48:30.706783Z","submitted_at":"2026-03-10T13:47:15Z","title":"EsoLang-Bench: Evaluating Genuine Reasoning in Large Language Models via Esoteric Programming Languages","version":2},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-05-15T13:32:51.545510Z"},"links":{"cited_paper":"/paper/2406.19470","citing_paper":"/paper/2603.09678"},"observation_digest":"sha256:0c7eb29dfe80ff94d997379132cbabbe2299f21218910a134341ce44611e8f4e","observation_id":"4f8cf4c5-2b88-49df-9087-696db27e308e","resolution":{"observed_at":"2026-05-15T13:35:51.808802Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19470","last_updated":"2024-11-11T02:27:54Z","snapshot_observed_at":"2026-07-06T18:38:09.490968Z","submitted_at":"2024-06-27T18:21:32Z","title":"Changing Answer Order Can Decrease MMLU Accuracy","version":2},"cited_work":{"arxiv_id":"2406.19470","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2406.19470","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Changing answer order can decrease MMLU accuracy.arXiv preprint arXiv:2406.19470","venue":null,"work_id":"9464f44f-c7eb-4b90-85b3-311c3aa8dc69","year":2024},"citing_paper":{"arxiv_id":"2605.07268","last_updated":"2026-05-08T05:33:58Z","snapshot_observed_at":"2026-07-06T23:19:41.053744Z","submitted_at":"2026-05-08T05:33:58Z","title":"From 0-Order Selection to 2-Order Judgment: Combinatorial Hardening Exposes Compositional Failures in Frontier LLMs","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-05-11T02:35:11.605353Z"},"links":{"cited_paper":"/paper/2406.19470","citing_paper":"/paper/2605.07268"},"observation_digest":"sha256:944823132837071664bab32ef2a21e69ad689d0988a2cff3215e6a243acde96a","observation_id":"372b4a57-b4d2-44ac-bc76-49c7410da6f5","resolution":{"observed_at":"2026-05-11T03:15:55.896876Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19470","last_updated":"2024-11-11T02:27:54Z","snapshot_observed_at":"2026-07-06T18:38:09.490968Z","submitted_at":"2024-06-27T18:21:32Z","title":"Changing Answer Order Can Decrease MMLU Accuracy","version":2},"cited_work":{"arxiv_id":"2406.19470","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2406.19470","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Changing answer order can decrease MMLU accuracy.arXiv preprint arXiv:2406.19470","venue":null,"work_id":"9464f44f-c7eb-4b90-85b3-311c3aa8dc69","year":2024},"citing_paper":{"arxiv_id":"2605.15607","last_updated":"2026-08-03T17:44:41Z","snapshot_observed_at":"2026-08-06T23:37:48.827947Z","submitted_at":"2026-05-15T04:37:31Z","title":"Syntax Without Semantics: Teaching Large Language Models to Code in an Unseen Language","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-05-20T19:12:42.655490Z"},"links":{"cited_paper":"/paper/2406.19470","citing_paper":"/paper/2605.15607"},"observation_digest":"sha256:3292f859b5c995846c03faf0bcb21fe4c19ce8f4858a68b2f66b18c781165b84","observation_id":"088feab2-b4fc-47ed-975b-e960f681f599","resolution":{"observed_at":"2026-05-20T19:13:40.350702Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19470","last_updated":"2024-11-11T02:27:54Z","snapshot_observed_at":"2026-07-06T18:38:09.490968Z","submitted_at":"2024-06-27T18:21:32Z","title":"Changing Answer Order Can Decrease MMLU Accuracy","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.19470","snapshot_observed_at":"2026-08-04T05:10:06.677483Z","title":"Changing answer order can decrease MMLU accuracy.arXiv preprint arXiv:2406.19470,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2605.15607","last_updated":"2026-08-03T17:44:41Z","snapshot_observed_at":"2026-08-06T23:37:48.827947Z","submitted_at":"2026-05-15T04:37:31Z","title":"Syntax Without Semantics: Teaching Large Language Models to Code in an Unseen Language","version":2},"reference_index":2025,"source":"pdf_text","source_observed_at":"2026-08-04T05:10:06.677483Z"},"links":{"cited_paper":"/paper/2406.19470","citing_paper":"/paper/2605.15607"},"observation_digest":"sha256:4f074ea1a56871efc2d27d4d8837cb361b7b1c2d6b9a825aab4eef4679cc8da7","observation_id":"31f1a6c9-2d62-42d4-9011-825f53ad4b26","resolution":{"observed_at":"2026-08-04T05:10:06.677483Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2406.19470/citation-record","integrity":"/paper/2406.19470/integrity","json":"/paper/2406.19470/citation-record.json","paper":"/paper/2406.19470"},"outbound":[],"paper":{"arxiv_id":"2406.19470","last_updated":"2024-11-11T02:27:54Z","latest_version":2,"primary_category":"cs.CL","snapshot_observed_at":"2026-07-06T18:38:09.490968Z","submitted_at":"2024-06-27T18:21:32Z","title":"Changing Answer Order Can Decrease MMLU Accuracy"},"reference_resolution":{"displayed":0,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":0,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":0},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"thesis":"As of 12 August 2026, this Paper Citation Record lists 0 of 0 outbound references and 12 inbound Pith citation observations for arXiv:2406.19470."}