{"as_of":"2026-08-18T03:00:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:b4abfb82d355012588321eaf86b506a2c9eec3557789647ff9c613e730169ca8","coverage":[{"denominator":70,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":70,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T17:21:06.866133Z","state":"measured"},{"denominator":75,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":75,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-17T06:30:58.91139+00:00","state":"measured"},{"denominator":5,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":5,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T14:19:01.315611Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-02T23:47:27.665680Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"cited_work":{"arxiv_id":"2508.13144","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2508.13144","snapshot_observed_at":"2026-07-02T23:47:27.665680Z","title":"Signal and noise: A framework for reducing uncertainty in language model evaluation.arXiv preprint arXiv:2508.13144, 2025","venue":null,"work_id":"79b5341a-c36a-445b-9ae3-4c9b795c213a","year":2025},"citing_paper":{"arxiv_id":"2606.07616","last_updated":"2026-05-29T20:55:15Z","snapshot_observed_at":"2026-07-06T23:47:15.041928Z","submitted_at":"2026-05-29T20:55:15Z","title":"Item Response Scaling Laws: A Measurement Theory Approach for Efficient and Generalizable Neural Scaling Estimation","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-06-28T22:48:20.193699Z"},"links":{"cited_paper":"/paper/2508.13144","citing_paper":"/paper/2606.07616"},"observation_digest":"sha256:d049d0bdb6eb4d4b96fffa3ac0b2220cb23fd108f9c40418bc2911d6141e8a0b","observation_id":"5836d679-f4c9-4ecd-9e88-f0cc6685728c","resolution":{"observed_at":"2026-06-28T22:52:45.619149Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"cited_work":{"arxiv_id":"2508.13144","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2508.13144","snapshot_observed_at":"2026-07-02T23:47:27.665680Z","title":"Signal and noise: A framework for reducing uncertainty in language model evaluation.arXiv preprint arXiv:2508.13144, 2025","venue":null,"work_id":"79b5341a-c36a-445b-9ae3-4c9b795c213a","year":2025},"citing_paper":{"arxiv_id":"2606.08679","last_updated":"2026-06-07T15:31:29Z","snapshot_observed_at":"2026-08-04T23:05:45.103241Z","submitted_at":"2026-06-07T15:31:29Z","title":"Rank Intervals for Leaderboards: A Hierarchical Framework for Model Evaluation","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-06-27T17:54:22.974336Z"},"links":{"cited_paper":"/paper/2508.13144","citing_paper":"/paper/2606.08679"},"observation_digest":"sha256:46749190559cdbaf1c409694d24b93085eca012dfd6264cfd002c90bf4ab762b","observation_id":"e143dbd7-b379-4fcb-8923-851e2092691d","resolution":{"observed_at":"2026-07-02T23:47:27.667294Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"cited_work":{"arxiv_id":"2508.13144","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2508.13144","snapshot_observed_at":"2026-07-02T23:47:27.665680Z","title":"Signal and noise: A framework for reducing uncertainty in language model evaluation.arXiv preprint arXiv:2508.13144, 2025","venue":null,"work_id":"79b5341a-c36a-445b-9ae3-4c9b795c213a","year":2025},"citing_paper":{"arxiv_id":"2606.28551","last_updated":"2026-08-10T01:17:41Z","snapshot_observed_at":"2026-08-13T23:28:37.542175Z","submitted_at":"2026-06-26T19:11:29Z","title":"DataComp-VLM: Improved Open Datasets for Vision-Language Models","version":1},"reference_index":97,"source":"pdf_text","source_observed_at":"2026-06-30T01:16:16.834861Z"},"links":{"cited_paper":"/paper/2508.13144","citing_paper":"/paper/2606.28551"},"observation_digest":"sha256:3f8f962c7110eb461bb4af800c22c358ae352442231e884215e5daf82df79f14","observation_id":"28aed6d8-f367-48d4-94ae-562d17e30cdc","resolution":{"observed_at":"2026-07-01T15:45:47.697916Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"cited_work":{"arxiv_id":"2508.13144","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2508.13144","snapshot_observed_at":"2026-07-02T23:47:27.665680Z","title":"Signal and noise: A framework for reducing uncertainty in language model evaluation.arXiv preprint arXiv:2508.13144, 2025","venue":null,"work_id":"79b5341a-c36a-445b-9ae3-4c9b795c213a","year":2025},"citing_paper":{"arxiv_id":"2606.28551","last_updated":"2026-08-10T01:17:41Z","snapshot_observed_at":"2026-08-13T23:28:37.542175Z","submitted_at":"2026-06-26T19:11:29Z","title":"DataComp-VLM: Improved Open Datasets for Vision-Language Models","version":2},"reference_index":97,"source":"pdf_text","source_observed_at":"2026-07-02T21:10:10.548489Z"},"links":{"cited_paper":"/paper/2508.13144","citing_paper":"/paper/2606.28551"},"observation_digest":"sha256:7bd08b74f1baa3dd306cde36616d03c3fd2224ca747ede140c3b71f7920bd0af","observation_id":"cb4686b8-c5f2-4d3c-b97c-306c93e5f914","resolution":{"observed_at":"2026-07-02T21:17:24.071487Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2508.13144","snapshot_observed_at":"2026-08-15T14:19:01.315611Z","title":"Heineman et al","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.11323","last_updated":"2026-08-11T18:16:51Z","snapshot_observed_at":"2026-08-17T10:50:43.521413Z","submitted_at":"2026-08-11T18:16:51Z","title":"Deployment Decision Reliability: A Generalizability-Theory Framework for Sizing Long-Horizon Agent Evaluations","version":1},"reference_index":1972,"source":"pdf_text","source_observed_at":"2026-08-15T14:19:01.315611Z"},"links":{"cited_paper":"/paper/2508.13144","citing_paper":"/paper/2608.11323"},"observation_digest":"sha256:6cf97022b17e025431bbe2f933e443192cc4e79f4f806e2de1260afc168d887d","observation_id":"9da7396a-b91e-48bb-8ee5-177e866cf05f","resolution":{"observed_at":"2026-08-15T14:19:01.315611Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2508.13144/citation-record","integrity":"/paper/2508.13144/integrity","json":"/paper/2508.13144/citation-record.json","paper":"/paper/2508.13144"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2108.07732","last_updated":"2021-08-16T03:57:30Z","snapshot_observed_at":"2026-08-15T17:40:38.050939Z","submitted_at":"2021-08-16T03:57:30Z","title":"Program Synthesis with Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.07732","snapshot_observed_at":"2026-08-15T17:21:06.602037Z","title":"Program synthesis with large language models","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.602037Z"},"links":{"cited_paper":"/paper/2108.07732","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:5b87f2b487ba34974ae645a94cc34fa8c7de11d1ddf9b21756e7df5b06277e1a","observation_id":"e05913ba-f248-470b-8ce7-89afcc758094","resolution":{"observed_at":"2026-08-15T17:21:06.602037Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.644467Z","title":"An empirical investigation of statistical significance in NLP","venue":null,"work_id":"d6652397-f8fe-4c5d-8741-068ae94cea4c","year":2012},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.607012Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:3496e48c5a5c84e41dcae7bf66ee5b53f2ebd3e5704b46cbb5422c5801649767","observation_id":"5bed2fbe-23cb-4657-8495-93c4718f35d4","resolution":{"observed_at":"2026-08-15T17:21:07.648346Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.04403","last_updated":"2025-08-22T17:27:09Z","snapshot_observed_at":"2026-08-17T15:54:07.298847Z","submitted_at":"2024-12-05T18:21:49Z","title":"Establishing Task Scaling Laws via Compute-Efficient Model Ladders","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.04403","snapshot_observed_at":"2026-08-15T17:21:06.610860Z","title":"Estab- lishing task scaling laws via compute-efficient model ladders","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.610860Z"},"links":{"cited_paper":"/paper/2412.04403","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:968b4aa1a4830922dc6eebcf766c605bab6e055f0966fa258f1bf2830933be6e","observation_id":"a3c03f77-dfbd-46e9-b42f-b5aad8f57ab6","resolution":{"observed_at":"2026-08-15T17:21:06.610860Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.633327Z","title":"Piqa: Reasoning about physical commonsense in natural language","venue":null,"work_id":"5584e5a6-d8e3-46db-8e5e-d5329d85c2c4","year":2020},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.615220Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:84c402af62996333231ea5708d1cd49f795f001847c3fcb36a40200730f9ef64","observation_id":"fdbc956b-b11b-446a-8cb6-66e797f35396","resolution":{"observed_at":"2026-08-15T17:21:07.637264Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.03476","last_updated":"2024-06-05T17:29:15Z","snapshot_observed_at":"2026-08-16T13:45:47.666351Z","submitted_at":"2024-06-05T17:29:15Z","title":"Does your data spark joy? Performance gains from domain upsampling at the end of training","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.03476","snapshot_observed_at":"2026-08-15T17:21:06.619163Z","title":"Larsen, Sean Owen, and Jonathan Frankle","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.619163Z"},"links":{"cited_paper":"/paper/2406.03476","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:501bce58ede9ec32fa4e2a3c2d9d8d2f23952a19d318a58db2e9573f1a0d60ee","observation_id":"b1035626-9902-496a-8b1b-58858639b199","resolution":{"observed_at":"2026-08-15T17:21:06.619163Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.01747","last_updated":"2025-05-28T15:29:03Z","snapshot_observed_at":"2026-08-16T12:53:30.968457Z","submitted_at":"2025-03-03T17:15:17Z","title":"Position: Don't Use the CLT in LLM Evals With Fewer Than a Few Hundred Datapoints","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.01747","snapshot_observed_at":"2026-08-15T17:21:06.623462Z","title":"Position: Don’t use the clt in llm evals with fewer than a few hundred datapoints","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.623462Z"},"links":{"cited_paper":"/paper/2503.01747","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:f6d3a970a7b98cd67496f9a37037d8b0cf0401ff8003258a8992090c1735f73c","observation_id":"9ecacb2e-2e88-4c62-b564-f43dbd4ca89c","resolution":{"observed_at":"2026-08-15T17:21:06.623462Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:06.627643Z","title":"With little power comes great responsibility","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.627643Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:edc2cc978564e555a1ce61cd21f4d2af9de8e1b021f86079053ea7eef9d4a933","observation_id":"67ec5ff3-50be-4280-ba5e-f485d514b922","resolution":{"observed_at":"2026-08-15T17:21:06.627643Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T11:58:24.516369Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-15T17:21:06.631354Z","title":"Evaluating large language models trained on code","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.631354Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:ea6342af9053fbabada54d6e4bc2629a77cd5f8e88a861a182fcf4ec4d3d7870","observation_id":"4f4b48fe-7993-4785-8546-de05fc20e669","resolution":{"observed_at":"2026-08-15T17:21:06.631354Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.11840","last_updated":"2025-06-02T19:33:41Z","snapshot_observed_at":"2026-08-16T13:09:04.911545Z","submitted_at":"2024-10-15T17:59:10Z","title":"A Hitchhiker's Guide to Scaling Law Estimation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.11840","snapshot_observed_at":"2026-08-15T17:21:06.635018Z","title":"A hitchhiker’s guide to scaling law estimation","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.635018Z"},"links":{"cited_paper":"/paper/2410.11840","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:15fb0d621b07445ba2a2ba3b82c6deeb2b26cfa38baf8b3d165bfce530cbace0","observation_id":"399d888d-29f1-4d1f-8c83-57a8af153571","resolution":{"observed_at":"2026-08-15T17:21:06.635018Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.623020Z","title":"Boolq: Exploring the surprising difficulty of natural yes/no questions","venue":null,"work_id":"a229c6c8-1ca8-4671-8601-0404cbbb76c0","year":2019},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.639188Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:ba3cd7697e0efdc4fbd073d9381b0ef4ac0a08bce8db20bbc31b79e58f66ba39","observation_id":"d74f2461-158f-444f-917f-42dc1d26a352","resolution":{"observed_at":"2026-08-15T17:21:07.626502Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1803.05457","last_updated":"2018-03-14T18:04:21Z","snapshot_observed_at":"2026-08-14T19:36:07.505691Z","submitted_at":"2018-03-14T18:04:21Z","title":"Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1803.05457","snapshot_observed_at":"2026-08-15T17:21:06.642868Z","title":"Think you have solved question answering? try arc, the ai2 reasoning challenge","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.642868Z"},"links":{"cited_paper":"/paper/1803.05457","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:c6ac7e7b8c338f3ea7fd13a20ca5214675a18ebac6ec5382b27cf7ee789aadc9","observation_id":"6d534e44-cf20-48f6-8125-ccec2d3e7422","resolution":{"observed_at":"2026-08-15T17:21:06.642868Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2110.14168","last_updated":"2021-11-18T00:23:45Z","snapshot_observed_at":"2026-08-14T02:43:01.480086Z","submitted_at":"2021-10-27T04:49:45Z","title":"Training Verifiers to Solve Math Word Problems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.14168","snapshot_observed_at":"2026-08-15T17:21:06.646553Z","title":"Training verifiers to solve math word problems","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.646553Z"},"links":{"cited_paper":"/paper/2110.14168","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:11862b40f257a1b34443c8f8002fa421557e4db683fc3eb2df00dd5498fcfe9c","observation_id":"71e8644f-a36e-4011-aaa5-e94db02090b5","resolution":{"observed_at":"2026-08-15T17:21:06.646553Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.612522Z","title":"Un- derspecification presents challenges for credibility in modern machine learning","venue":null,"work_id":"e5b6b4a5-75bd-4dd7-8ca2-1d5bcf511bed","year":2022},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.650787Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:9cfc49a511856bf67e0ad16cfa7c00037bd4699dac5514848d09678633c0275b","observation_id":"6f107d14-e154-4053-9744-c53c6b3497de","resolution":{"observed_at":"2026-08-15T17:21:07.616189Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2002.06305","last_updated":"2020-02-15T02:40:10Z","snapshot_observed_at":"2026-08-12T00:45:25.809655Z","submitted_at":"2020-02-15T02:40:10Z","title":"Fine-Tuning Pretrained Language Models: Weight Initializations, Data Orders, and Early Stopping","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2002.06305","snapshot_observed_at":"2026-08-15T17:21:06.654290Z","title":"Fine-tuning pretrained language models: Weight initializations, data orders, and early stopping","venue":null,"work_id":null,"year":2002},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.654290Z"},"links":{"cited_paper":"/paper/2002.06305","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:384ea38024d88923181ccb1399ad163e6b86c04ffe5545b70ee129a32083b125","observation_id":"b2512895-8f04-436e-91a2-f72e947e82d9","resolution":{"observed_at":"2026-08-15T17:21:06.654290Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.15796","last_updated":"2025-01-15T02:48:59Z","snapshot_observed_at":"2026-08-16T14:07:07.590212Z","submitted_at":"2024-03-23T11:03:31Z","title":"Understanding Emergent Abilities of Language Models from the Loss Perspective","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.15796","snapshot_observed_at":"2026-08-15T17:21:06.657960Z","title":"Understanding emergent abilities of language models from the loss perspective","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.657960Z"},"links":{"cited_paper":"/paper/2403.15796","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:587e33a3dc7f95b5ceb2cbc03af37f885d4d43aefbb2f498e0ec75caa077b4c0","observation_id":"12920284-d0f9-4e04-a45f-19beed62d836","resolution":{"observed_at":"2026-08-15T17:21:06.657960Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.600182Z","title":"Drop: A reading comprehension benchmark requiring discrete reasoning over paragraphs","venue":null,"work_id":"e29efe52-1f56-423e-a721-645edb1174fe","year":2019},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.661755Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:ace5d3697eee059fbdc132885f72a5fa358a86c0fb06c3c25b3a6b5ca893e7d2","observation_id":"849ef073-95a7-4bbc-a4b5-9b742438d1f2","resolution":{"observed_at":"2026-08-15T17:21:07.604615Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21783","last_updated":"2024-11-23T23:27:33Z","snapshot_observed_at":"2026-08-13T17:20:44.002518Z","submitted_at":"2024-07-31T17:54:27Z","title":"The Llama 3 Herd of Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21783","snapshot_observed_at":"2026-08-15T17:21:06.665186Z","title":"The llama 3 herd of models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.665186Z"},"links":{"cited_paper":"/paper/2407.21783","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:acd95cd8bca7e1b310d70135cd9d727c966855e06cec40aa15290b86f5196278","observation_id":"bfd72767-cbdc-4198-aad9-74f43d1e77d0","resolution":{"observed_at":"2026-08-15T17:21:06.665186Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:06.668985Z","title":"Open llm leaderboard v2","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.668985Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:1831a3400ecf16d4d52e9b759f10d39aa5e6313781b43f8238da1e98095337fd","observation_id":"7debad2f-c7a6-4d3a-98aa-4ffde8f26d3c","resolution":{"observed_at":"2026-08-15T17:21:06.668985Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.08540","last_updated":"2024-06-14T20:21:05Z","snapshot_observed_at":"2026-08-16T14:10:11.362149Z","submitted_at":"2024-03-13T13:54:00Z","title":"Language models scale reliably with over-training and on downstream tasks","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.08540","snapshot_observed_at":"2026-08-15T17:21:06.672554Z","title":"Language models scale reliably with over-training and on downstream tasks","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.672554Z"},"links":{"cited_paper":"/paper/2403.08540","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:4d391e9a1fb9ff5e7931668bd5268dc29bfd465ee85a3054ef49172223caae47","observation_id":"7b6af821-46f4-4882-aa93-fa37548a0574","resolution":{"observed_at":"2026-08-15T17:21:06.672554Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2101.00027","last_updated":"2020-12-31T19:00:10Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2020-12-31T19:00:10Z","title":"The Pile: An 800GB Dataset of Diverse Text for Language Modeling","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2101.00027","snapshot_observed_at":"2026-08-15T17:21:06.676181Z","title":"The pile: An 800gb dataset of diverse text for language modeling","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.676181Z"},"links":{"cited_paper":"/paper/2101.00027","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:09601f6222f3cfe32af11bafcdc0951b35d7056dfca16d34bc2cb64d7ca9290f","observation_id":"caa45890-2edc-4fcc-8a50-4ad35b422e7a","resolution":{"observed_at":"2026-08-15T17:21:06.676181Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.04127","last_updated":"2025-01-10T14:31:21Z","snapshot_observed_at":"2026-08-16T13:45:30.496997Z","submitted_at":"2024-06-06T14:49:06Z","title":"Are We Done with MMLU?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.04127","snapshot_observed_at":"2026-08-15T17:21:06.679877Z","title":"Are we done with mmlu? arXiv preprint arXiv:2406.04127, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.679877Z"},"links":{"cited_paper":"/paper/2406.04127","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:6a7a0d81346357dc2d959ab9c3d0a77e4f0ac710c57c3507ba3639886878007a","observation_id":"6c1b5926-b73b-485e-8ead-9f20cf4e7070","resolution":{"observed_at":"2026-08-15T17:21:06.679877Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.08446","last_updated":"2025-02-11T18:59:26Z","snapshot_observed_at":"2026-08-16T13:43:38.372862Z","submitted_at":"2024-06-12T17:37:09Z","title":"OLMES: A Standard for Language Model Evaluations","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.08446","snapshot_observed_at":"2026-08-15T17:21:06.683770Z","title":"Olmes: A standard for language model evaluations","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.683770Z"},"links":{"cited_paper":"/paper/2406.08446","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:70dc1fd566cb79f88e96dfbe3557e3d8c3ddd1d3a3fd2eb6ecad5c5fdac610a5","observation_id":"eacdda79-bdab-4511-b107-0f9eb807efe3","resolution":{"observed_at":"2026-08-15T17:21:06.683770Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.03300","last_updated":"2021-01-12T18:57:11Z","snapshot_observed_at":"2026-08-13T20:44:28.824685Z","submitted_at":"2020-09-07T17:59:25Z","title":"Measuring Massive Multitask Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.03300","snapshot_observed_at":"2026-08-15T17:21:06.687583Z","title":"Measuring massive multitask language understanding","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.687583Z"},"links":{"cited_paper":"/paper/2009.03300","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:d1608767fd491d78bcf3b76513fdac9a4c8b4d6f08eac7d300f46d29b6ba1a65","observation_id":"6fff7c1b-047d-497e-a0e2-c18809fa2ee9","resolution":{"observed_at":"2026-08-15T17:21:06.687583Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2203.15556","last_updated":"2022-03-29T13:38:03Z","snapshot_observed_at":"2026-08-09T19:52:33.533277Z","submitted_at":"2022-03-29T13:38:03Z","title":"Training Compute-Optimal Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2203.15556","snapshot_observed_at":"2026-08-15T17:21:06.691083Z","title":"Training compute-optimal large language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.691083Z"},"links":{"cited_paper":"/paper/2203.15556","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:eaaaaf27f6ae06fdd4e69fe50773bfef131c9e8bf105656d392a3bdac82819bc","observation_id":"e42f60ea-7c58-44ff-bc85-99b4c3a4d298","resolution":{"observed_at":"2026-08-15T17:21:06.691083Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.09937","last_updated":"2024-08-19T13:55:42Z","snapshot_observed_at":"2026-08-16T14:00:45.702335Z","submitted_at":"2024-04-15T17:03:41Z","title":"Compression Represents Intelligence Linearly","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.09937","snapshot_observed_at":"2026-08-15T17:21:06.694837Z","title":"Compression represents intelligence linearly","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.694837Z"},"links":{"cited_paper":"/paper/2404.09937","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:6f296f1570a11c43b1e9d2f63a88d887b547b1d188fa959202c1e2823a562486","observation_id":"759da69e-87db-4833-8e14-b6a9b1819c95","resolution":{"observed_at":"2026-08-15T17:21:06.694837Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.581921Z","title":"Triviaqa: A large scale distantly supervised challenge dataset for reading comprehension","venue":null,"work_id":"eb80b749-91cf-49b2-95f0-afc25ccfc68e","year":2017},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.698824Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:87dcd1461d1a85a72196e6b46510ac96be4e0ea642b56cb26d3aceed22163373","observation_id":"b0faf913-cc1d-4d17-8d23-7038fcbda701","resolution":{"observed_at":"2026-08-15T17:21:07.586179Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2001.08361","last_updated":"2020-01-23T03:59:20Z","snapshot_observed_at":"2026-08-13T17:41:53.092611Z","submitted_at":"2020-01-23T03:59:20Z","title":"Scaling Laws for Neural Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2001.08361","snapshot_observed_at":"2026-08-15T17:21:06.702386Z","title":"Scaling laws for neural language models","venue":null,"work_id":null,"year":2001},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.702386Z"},"links":{"cited_paper":"/paper/2001.08361","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:19d84d42fa2e6eaa029a69678d5e7e64ed5e9087c36d432f92dd4105a9f30f3a","observation_id":"c92b553a-ee50-4228-a748-5a746c658b45","resolution":{"observed_at":"2026-08-15T17:21:06.702386Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.569808Z","title":"Natural questions: A benchmark for question answering research","venue":null,"work_id":"885902c9-68aa-4c7d-9096-7b323b1b77fd","year":2019},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.706024Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:b63b548aae33e53033bc035d063b8fb04ea05137e61a4fd62455c66b2532a26c","observation_id":"09caf088-d013-419e-b24f-aabf172c6bd2","resolution":{"observed_at":"2026-08-15T17:21:07.574010Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.557040Z","title":"Finetasks: Finding signal in a haystack of 200+ multilingual tasks, 2024","venue":null,"work_id":"d53a58a6-a689-49e4-90c2-94ad98d750e5","year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.709638Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:09449f840ede8193f650228f18f7bbe1f5aa7be99ff4b0d19a87162089c88646","observation_id":"6fcf2374-916f-40f2-92c3-b7af8d0e773c","resolution":{"observed_at":"2026-08-15T17:21:07.561859Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2206.14858","last_updated":"2022-07-01T02:15:12Z","snapshot_observed_at":"2026-08-17T10:54:45.389424Z","submitted_at":"2022-06-29T18:54:49Z","title":"Solving Quantitative Reasoning Problems with Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2206.14858","snapshot_observed_at":"2026-08-15T17:21:06.713094Z","title":"Solving quantitative reasoning problems with language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.713094Z"},"links":{"cited_paper":"/paper/2206.14858","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:c0c010ea6b54b7ee7962ac97e07f2bb47f66eac10474e71c6265ecc92137c56a","observation_id":"65d90a86-b989-419c-92b8-db89f68b9bff","resolution":{"observed_at":"2026-08-15T17:21:06.713094Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:06.716687Z","title":"Datacomp-lm: In search of the next generation of training sets for language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.716687Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:823dee8ebbeed9532579aeed245c381574b8a8d64d538551428bc6e4926a2ff3","observation_id":"98757a67-a984-4d43-b884-dd5987894c53","resolution":{"observed_at":"2026-08-15T17:21:06.716687Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.08351","last_updated":"2025-02-28T08:14:49Z","snapshot_observed_at":"2026-08-16T13:35:08.798583Z","submitted_at":"2024-07-11T10:03:47Z","title":"AutoBencher: Towards Declarative Benchmark Construction","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.08351","snapshot_observed_at":"2026-08-15T17:21:06.720504Z","title":"Autobencher: Creating salient, novel, difficult datasets for language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.720504Z"},"links":{"cited_paper":"/paper/2407.08351","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:82aa77974121a3a74cb0895215aeab54b085ea06394b2ff4b3d269213b75f85c","observation_id":"aa6ef03c-ca95-4aad-8484-a58bc1bed9f9","resolution":{"observed_at":"2026-08-15T17:21:06.720504Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-15T17:21:06.724061Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.724061Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:823ccea90a2744e44e6afaf0c691105fba46f138e6b9f972b5fbca4e4e97d212","observation_id":"e98de564-0e70-4497-adb2-4db9a41f2a3f","resolution":{"observed_at":"2026-08-15T17:21:06.724061Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:06.727477Z","title":"Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.727477Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:0dcc9f8aace601fbb6d491ae85f28a7f2cbf5b9ffc940f20db6169e4f62bbcd7","observation_id":"c82f593d-909f-402b-8ef2-7b9b487607c0","resolution":{"observed_at":"2026-08-15T17:21:06.727477Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.01492","last_updated":"2025-01-23T17:35:43Z","snapshot_observed_at":"2026-08-16T13:38:01.910968Z","submitted_at":"2024-07-01T17:31:03Z","title":"RegMix: Data Mixture as Regression for Language Model Pre-training","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.01492","snapshot_observed_at":"2026-08-15T17:21:06.730725Z","title":"Regmix: Data mixture as regression for language model pre-training","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.730725Z"},"links":{"cited_paper":"/paper/2407.01492","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:d6b159330af82f336215d83962661f0c8cee16726f7a1e1e89ff0e28919342b5","observation_id":"43ea5e26-5e14-4592-b5f7-3f0a0caa4e0f","resolution":{"observed_at":"2026-08-15T17:21:06.730725Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.10229","last_updated":"2024-06-14T17:59:54Z","snapshot_observed_at":"2026-08-16T13:42:51.314941Z","submitted_at":"2024-06-14T17:59:54Z","title":"Quantifying Variance in Evaluation Benchmarks","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.10229","snapshot_observed_at":"2026-08-15T17:21:06.734304Z","title":"Quantifying variance in evaluation bench- marks","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.734304Z"},"links":{"cited_paper":"/paper/2406.10229","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:55f33caa74e2b53ca5f6ee3c311231859c9e9cbab6b9f20ab9a071c9a22e81f0","observation_id":"f30632b9-3531-4029-9b0c-61a24be3029f","resolution":{"observed_at":"2026-08-15T17:21:06.734304Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.10523","last_updated":"2024-12-07T20:22:22Z","snapshot_observed_at":"2026-08-16T14:34:07.692350Z","submitted_at":"2023-12-16T19:12:45Z","title":"Paloma: A Benchmark for Evaluating Language Model Fit","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.10523","snapshot_observed_at":"2026-08-15T17:21:06.737675Z","title":"Paloma: A benchmark for evaluating language model fit","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.737675Z"},"links":{"cited_paper":"/paper/2312.10523","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:8de50e22c2c252792885e3f2b201c86858aa33ae81e1f9031d80cdf55b17c287","observation_id":"baa20cb0-0ce7-45a0-8161-c2b529ff449b","resolution":{"observed_at":"2026-08-15T17:21:06.737675Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.531733Z","title":"Hwang, Luca Soldaini, Akshita Bhagia, Jiacheng Liu, Dirk Groeneveld, Oyvind Tafjord, Noah A","venue":null,"work_id":"034cf403-f4f1-47be-8665-c3466773fd29","year":2025},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.741476Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:11a402c82befa5d1fca2cc4c1f5ad522fead960b19a12fc52e8545a2e72a1e44","observation_id":"a67e8339-a73f-44f7-ac98-7979e9858feb","resolution":{"observed_at":"2026-08-15T17:21:07.535539Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:06.745035Z","title":"Can a suit of armor conduct electricity? a new dataset for open book question answering","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.745035Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:1e2784f9e47eb3e249f7906e58200bb0cc9718aa3ce25b6f874f2e4943793fbc","observation_id":"2c6cce69-e09c-4b38-8ec7-892d8b10648d","resolution":{"observed_at":"2026-08-15T17:21:06.745035Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.00640","last_updated":"2024-11-01T14:57:16Z","snapshot_observed_at":"2026-08-17T04:34:26.881993Z","submitted_at":"2024-11-01T14:57:16Z","title":"Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.00640","snapshot_observed_at":"2026-08-15T17:21:06.748522Z","title":"Adding error bars to evals: A statistical approach to language model evaluations","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.748522Z"},"links":{"cited_paper":"/paper/2411.00640","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:1c4f720b6fc821c787b8eb081d0674f1cf74d4ea4ee27cc00d8a362117bf60c0","observation_id":"2b0035f8-99b7-4487-a703-a34fe4963d10","resolution":{"observed_at":"2026-08-15T17:21:06.748522Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.05229","last_updated":"2025-08-27T16:24:39Z","snapshot_observed_at":"2026-08-16T08:54:56.543625Z","submitted_at":"2024-10-07T17:36:37Z","title":"GSM-Symbolic: Understanding the Limitations of Mathematical Reasoning in Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.05229","snapshot_observed_at":"2026-08-15T17:21:06.752982Z","title":"Gsm-symbolic: Understanding the limitations of mathematical reasoning in large language models","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.752982Z"},"links":{"cited_paper":"/paper/2410.05229","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:a0af0da5163786b66515b4d719f6b26ee1359071a51c554b0094bc13b91ef812","observation_id":"838eefdc-f0c6-4ddd-9cd8-1d4b4a766fee","resolution":{"observed_at":"2026-08-15T17:21:06.752982Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.00656","last_updated":"2025-10-08T07:50:45Z","snapshot_observed_at":"2026-08-17T13:58:40.683829Z","submitted_at":"2024-12-31T21:55:10Z","title":"2 OLMo 2 Furious","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.00656","snapshot_observed_at":"2026-08-15T17:21:06.757147Z","title":"2 olmo 2 furious","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.757147Z"},"links":{"cited_paper":"/paper/2501.00656","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:9af265870792bddd8a3408b2b91177170d59b18040170a9309fed09d086ed752","observation_id":"617c3e2f-5a1d-4520-a575-3613e1b2bccf","resolution":{"observed_at":"2026-08-15T17:21:06.757147Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.513518Z","title":"Medmcqa: A large-scale multi-subject multi-choice dataset for medical domain question answering","venue":null,"work_id":"6617dea3-6209-4641-92be-09a447210e71","year":2022},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.760881Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:7d33c74ecab7a610166110eca1d343fdb853b9ac2a224ad5d7278293719464f1","observation_id":"95ab2b1f-7a23-473b-b5df-61ae2d3b73fc","resolution":{"observed_at":"2026-08-15T17:21:07.517487Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.12907","last_updated":"2024-11-21T12:17:22Z","snapshot_observed_at":"2026-08-16T13:43:45.443489Z","submitted_at":"2024-06-12T13:30:48Z","title":"Reconciling Kaplan and Chinchilla Scaling Laws","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.12907","snapshot_observed_at":"2026-08-15T17:21:06.764274Z","title":"Reconciling kaplan and chinchilla scaling laws","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.764274Z"},"links":{"cited_paper":"/paper/2406.12907","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:2ed412193f0ac64eff139acf03d15ee39e216c5eefbbc733c620892e3c00a514","observation_id":"27abd2b6-a631-4563-bf5d-8cddd35aee25","resolution":{"observed_at":"2026-08-15T17:21:06.764274Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.14992","last_updated":"2024-05-26T22:27:23Z","snapshot_observed_at":"2026-08-17T02:59:38.954198Z","submitted_at":"2024-02-22T22:05:23Z","title":"tinyBenchmarks: evaluating LLMs with fewer examples","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.14992","snapshot_observed_at":"2026-08-15T17:21:06.769157Z","title":"tinybenchmarks: evaluating llms with fewer examples","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.769157Z"},"links":{"cited_paper":"/paper/2402.14992","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:f5ddb7c69b0664c653c78c27296faa39475b385ae43ab0c928c2115ffd9111f7","observation_id":"2a652f7d-4536-4514-8b40-cfeab25c0d00","resolution":{"observed_at":"2026-08-15T17:21:06.769157Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.17681","last_updated":"2024-06-26T15:21:49Z","snapshot_observed_at":"2026-08-17T18:36:03.670938Z","submitted_at":"2024-06-25T16:13:53Z","title":"VarBench: Robust Language Model Benchmarking Through Dynamic Variable Perturbation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.17681","snapshot_observed_at":"2026-08-15T17:21:06.773082Z","title":"Varbench: Robust language model benchmarking through dynamic variable perturbation","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.773082Z"},"links":{"cited_paper":"/paper/2406.17681","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:a2304071a8a0393d596c819b282cfa9a8dadca65a647abb7edb4d1752d104468","observation_id":"cb7de8f9-3a79-4321-8963-74a5d054c843","resolution":{"observed_at":"2026-08-15T17:21:06.773082Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:06.776888Z","title":"Squad: 100,000+ questions for machine comprehension of text","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.776888Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:1e101b05e17714dc5d613383bd0f777a97e3c63f77f17099bd19089dcf3a0c9d","observation_id":"ee5b7608-cf6c-42c0-a1c5-5987eb979b4f","resolution":{"observed_at":"2026-08-15T17:21:06.776888Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.493294Z","title":null,"venue":null,"work_id":"8a88ce13-b8bc-4c58-a055-85e235762677","year":2019},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.780378Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:78ad9c18ee58d370bc891166d226c450a263e9d9d2cae2e02728a8d8ce3124b3","observation_id":"5eec718c-2a99-4bbf-be7c-768780257b2a","resolution":{"observed_at":"2026-08-15T17:21:07.497106Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2311.12022","last_updated":"2023-11-20T18:57:34Z","snapshot_observed_at":"2026-08-16T07:06:07.822225Z","submitted_at":"2023-11-20T18:57:34Z","title":"GPQA: A Graduate-Level Google-Proof Q&A Benchmark","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.12022","snapshot_observed_at":"2026-08-15T17:21:06.784311Z","title":"Stickland, Jackson Petty, Richard Yuanzhe Pang, Julien Dirani, Julian Michael, and Samuel R","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.784311Z"},"links":{"cited_paper":"/paper/2311.12022","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:f30b2942f9eaf4d1a095bf1290a880d812c039c17d374586c377ee8f58cca01b","observation_id":"cc21d9c2-fd19-4815-bfeb-ed14ac6b8347","resolution":{"observed_at":"2026-08-15T17:21:06.784311Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.10061","last_updated":"2025-06-13T21:16:23Z","snapshot_observed_at":"2026-08-16T12:50:30.166650Z","submitted_at":"2025-03-13T05:21:22Z","title":"Compute Optimal Scaling of Skills: Knowledge vs Reasoning","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.10061","snapshot_observed_at":"2026-08-15T17:21:06.788592Z","title":"Com- pute optimal scaling of skills: Knowledge vs reasoning","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.788592Z"},"links":{"cited_paper":"/paper/2503.10061","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:1f7b04ed213771d0940211f01f3cf569a9bb0fef41328592f4e4afcf93ee1e49","observation_id":"7392c061-e889-4e90-8d64-3eed331ce2fe","resolution":{"observed_at":"2026-08-15T17:21:06.788592Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.479833Z","title":"Observational scaling laws and the predictability of langauge model performance","venue":null,"work_id":"739c1941-dc72-4c01-989e-09631288b0dc","year":2025},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.792266Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:8b678a460783f27a439d73c7deb5dc046870b86f44936153fa0fe6fa19dde532","observation_id":"a4d2d35a-d8a7-4a6f-aa61-f6a0224d030c","resolution":{"observed_at":"2026-08-15T17:21:07.484796Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.461185Z","title":"Winogrande: An adversarial winograd schema challenge at scale","venue":null,"work_id":"7707d673-7b83-4893-8912-734dfe6990cb","year":2020},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.796613Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:9b6a1d6cceae08653019c60d3200b5d0239f0615c1fee3cea24ff36fd538afd2","observation_id":"32ddd0d9-6f4d-4ce6-bf56-80815b7030da","resolution":{"observed_at":"2026-08-15T17:21:07.465361Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.448240Z","title":"Social iqa: Commonsense reasoning about social interactions","venue":null,"work_id":"f4582fd3-f7a0-4632-94d6-344e973c57bf","year":2019},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.800208Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:24ed6ff133e3d2c5013b9edd22c74ff5adaa350089a9bff544182a628655e7a9","observation_id":"c0a280ef-0a83-49ee-92a1-93eeeb8c8f34","resolution":{"observed_at":"2026-08-15T17:21:07.452734Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.04391","last_updated":"2025-02-05T17:44:38Z","snapshot_observed_at":"2026-08-16T13:45:25.499566Z","submitted_at":"2024-06-06T17:46:56Z","title":"Why Has Predicting Downstream Capabilities of Frontier AI Models with Scale Remained Elusive?","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.04391","snapshot_observed_at":"2026-08-15T17:21:06.804001Z","title":"Why has predicting downstream capabilities of frontier ai models with scale remained elusive? arXiv preprint arXiv:2406.04391, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.804001Z"},"links":{"cited_paper":"/paper/2406.04391","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:7b7c2604d3c36303fe99ec95bb39f1fd923b9173b3a6a4feabc60e649faa417c","observation_id":"39f6ecb1-ebb6-4953-90e6-af124b5c14eb","resolution":{"observed_at":"2026-08-15T17:21:06.804001Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.11324","last_updated":"2024-07-01T22:28:01Z","snapshot_observed_at":"2026-08-12T15:05:30.306144Z","submitted_at":"2023-10-17T15:03:30Z","title":"Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.11324","snapshot_observed_at":"2026-08-15T17:21:06.808213Z","title":"Quantifying language models’ sensitivity to spurious features in prompt design or: How i learned to start worrying about prompt formatting, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.808213Z"},"links":{"cited_paper":"/paper/2310.11324","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:51c1e2d7cce1eb96669c9735829ac0a39f39d6005b88f3c28b46b390ee04dc4a","observation_id":"685cb3a3-87e0-4fc2-8c59-257766f9a5a6","resolution":{"observed_at":"2026-08-15T17:21:06.808213Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.00808","last_updated":"2025-08-02T08:34:03Z","snapshot_observed_at":"2026-08-16T12:53:53.790354Z","submitted_at":"2025-03-02T09:21:28Z","title":"Predictive Data Selection: The Data That Predicts Is the Data That Teaches","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.00808","snapshot_observed_at":"2026-08-15T17:21:06.812208Z","title":"Predictive data selection: The data that predicts is the data that teaches","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.812208Z"},"links":{"cited_paper":"/paper/2503.00808","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:a1f3f515d5d7cc923503e29e4ce4f8ef8fc280be7f31da3b5bdfd7febb277e37","observation_id":"04727587-5435-47c6-9a54-2a8a9b07f2fd","resolution":{"observed_at":"2026-08-15T17:21:06.812208Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.16035","last_updated":"2024-11-25T01:48:09Z","snapshot_observed_at":"2026-08-16T15:21:12.913921Z","submitted_at":"2024-11-25T01:48:09Z","title":"Predicting Emergent Capabilities by Finetuning","version":1},"cited_work":{"arxiv_id":"2411.16035","doi":null,"metadata_source":"pith","pith_arxiv_id":"2411.16035","snapshot_observed_at":"2026-08-15T17:21:06.986406Z","title":"Predicting Emergent Capabilities by Finetuning","venue":"cs.LG","work_id":"3f40812e-5a2b-43c1-815b-a7f78639563c","year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.816340Z"},"links":{"cited_paper":"/paper/2411.16035","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:15e36a550011be839dcf81b3b81c269739afad5cb857ddc13819fe118490d5b4","observation_id":"6ee9b21c-bb66-41d5-bc41-4c5df5b00050","resolution":{"observed_at":"2026-08-15T17:21:06.992504Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2206.04615","last_updated":"2023-06-12T17:51:15Z","snapshot_observed_at":"2026-07-06T13:19:12.109592Z","submitted_at":"2022-06-09T17:05:34Z","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2206.04615","snapshot_observed_at":"2026-08-15T17:21:06.821162Z","title":"Beyond the imitation game: Quantifying and extrapolating the capabilities of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.821162Z"},"links":{"cited_paper":"/paper/2206.04615","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:a870f09d651199196ebeb5724fc41687f1cc75592205e09af030a0ac2cdcc79c","observation_id":"eb372f24-0521-4aaa-b5ff-3b5628717d1e","resolution":{"observed_at":"2026-08-15T17:21:06.821162Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.434362Z","title":"Commonsenseqa: A question answering challenge targeting commonsense knowledge","venue":null,"work_id":"73631d83-ab45-4861-8f5a-b1bbd5cca978","year":2019},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.825725Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:7a2e47eed94cdb116b9b4a22e28f7e92c8538ecf9fcfa4cf28418cd1ce919bff","observation_id":"8ef02911-d648-40d4-bcdf-fdbbcd2c706e","resolution":{"observed_at":"2026-08-15T17:21:07.439453Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2307.09288","last_updated":"2023-07-19T17:08:59Z","snapshot_observed_at":"2026-08-07T12:56:43.323460Z","submitted_at":"2023-07-18T14:31:57Z","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.09288","snapshot_observed_at":"2026-08-15T17:21:06.829382Z","title":"Llama 2: Open foundation and fine-tuned chat models, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.829382Z"},"links":{"cited_paper":"/paper/2307.09288","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:0b8ccd6f93ebe48dffd4f83bc13ddcd85c5b87cc039d126ec76db1e868b7e092","observation_id":"372e5ce7-554d-48dd-98f9-629558701eda","resolution":{"observed_at":"2026-08-15T17:21:06.829382Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.421365Z","title":"200,000+ jeopardy! questions","venue":null,"work_id":"d580957c-83bc-412a-b3c4-68bc3cfcd6d8","year":2019},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.833155Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:788d9495c6aba9d668da7e004013199098e89908d4ee83e31e4f557040a90b5a","observation_id":"ed837778-35c6-4578-94e9-b864220a6874","resolution":{"observed_at":"2026-08-15T17:21:07.426367Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2502.03461","last_updated":"2025-02-05T18:58:19Z","snapshot_observed_at":"2026-08-10T16:22:07.766175Z","submitted_at":"2025-02-05T18:58:19Z","title":"Do Large Language Model Benchmarks Test Reliability?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.03461","snapshot_observed_at":"2026-08-15T17:21:06.836637Z","title":"Do large language model benchmarks test reliability? arXiv preprint arXiv:2502.03461, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.836637Z"},"links":{"cited_paper":"/paper/2502.03461","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:809dc3374c0adfa80e79256eff782c906563c8d0d82987f5c841eb789af93841","observation_id":"a4dd801b-6fec-4047-856c-bb21901d5adb","resolution":{"observed_at":"2026-08-15T17:21:06.836637Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:07.408417Z","title":"Wang, Alex Gu, Lovish Madaan, Dieuwke Hupkes, Jiawei Liu, Yuxiang Wei, Naman Jain, Yuhang Lai, Sten Sootla, Ofir Press, Baptiste Rozière, and Gabriel Synnaeve","venue":null,"work_id":"9adaa54e-554f-42c9-843e-2c903b434dbd","year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.840271Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:15618a685b7165f6c8aef997b4591e72be021d86d77eb00ca3b8a85de42baa5f","observation_id":"8e3817de-6df9-4aa0-b7b4-89753cc2fb02","resolution":{"observed_at":"2026-08-15T17:21:07.412450Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:06.843954Z","title":"Mmlu-pro: A more robust and challenging multi-task language understanding benchmark","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":64,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.843954Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:3040d7125e295c04e138e2fbf4b2f67ef326a74e0f2069c10529f74fe77f892d","observation_id":"1366c2c9-b3ff-4405-befe-c5117d9a1edf","resolution":{"observed_at":"2026-08-15T17:21:06.843954Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2206.07682","last_updated":"2022-10-26T05:06:24Z","snapshot_observed_at":"2026-08-17T00:14:01.413100Z","submitted_at":"2022-06-15T17:32:01Z","title":"Emergent Abilities of Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2206.07682","snapshot_observed_at":"2026-08-15T17:21:06.847630Z","title":"Emergent abilities of large language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":65,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.847630Z"},"links":{"cited_paper":"/paper/2206.07682","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:a1e21a77b72ccdaefa0841cd73d58f12786b84590384ae1276bc5bc5c293eaf8","observation_id":"7ee62304-b34c-4477-b14b-5f925c0dd973","resolution":{"observed_at":"2026-08-15T17:21:06.847630Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.10341","last_updated":"2025-07-16T07:09:29Z","snapshot_observed_at":"2026-08-14T15:17:42.222016Z","submitted_at":"2025-02-14T18:02:37Z","title":"Organize the Web: Constructing Domains Enhances Pre-Training Data Curation","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.10341","snapshot_observed_at":"2026-08-15T17:21:06.851539Z","title":"Organize the web: Constructing domains enhances pre-training data curation","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":66,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.851539Z"},"links":{"cited_paper":"/paper/2502.10341","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:acbafcb476c19549e0c6084615a9a025e4c796ed63e1871dd8cfd5e2e03fb9af","observation_id":"7baf326e-4d8a-48b6-8546-a51bfaddc91e","resolution":{"observed_at":"2026-08-15T17:21:06.851539Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.15018","last_updated":"2025-03-07T22:41:19Z","snapshot_observed_at":"2026-08-16T13:32:26.123645Z","submitted_at":"2024-07-21T00:10:23Z","title":"Answer, Assemble, Ace: Understanding How LMs Answer Multiple Choice Questions","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.15018","snapshot_observed_at":"2026-08-15T17:21:06.855163Z","title":"Answer, assemble, ace: Understanding how transformers answer multiple choice questions","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":67,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.855163Z"},"links":{"cited_paper":"/paper/2407.15018","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:c7f6017e69ff6f2b2adf6f01d67f253cd1ec2ae5d1a327031c7014648ad63fb5","observation_id":"2cd86841-44aa-4f4a-8e0c-0c1f4b79bc3d","resolution":{"observed_at":"2026-08-15T17:21:06.855163Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2203.03466","last_updated":"2022-03-28T08:12:14Z","snapshot_observed_at":"2026-08-16T17:16:25.053180Z","submitted_at":"2022-03-07T15:37:35Z","title":"Tensor Programs V: Tuning Large Neural Networks via Zero-Shot Hyperparameter Transfer","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2203.03466","snapshot_observed_at":"2026-08-15T17:21:06.858923Z","title":"Hu, Igor Babuschkin, Szymon Sidor, Xiaodong Liu, David Farhi, Nick Ryder, Jakub Pachocki, Weizhu Chen, and Jianfeng Gao","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":68,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.858923Z"},"links":{"cited_paper":"/paper/2203.03466","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:e8b858b23dd128aa6b062cf43223becc68839506dda156b17ad559e2d76aa343","observation_id":"e4ece66d-d26b-492a-9649-4306861d56fc","resolution":{"observed_at":"2026-08-15T17:21:06.858923Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T17:21:06.862717Z","title":"Hellaswag: Can a machine really finish your sentence? In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, pages 4791–4800, 2019","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":69,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.862717Z"},"links":{"citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:00d1f4e102ee87e25710a8955bcfa93f460a3e2b18165e0f37712c7b10a2e099","observation_id":"8d8b5102-915b-4c9c-99ad-3fa46c50b6de","resolution":{"observed_at":"2026-08-15T17:21:06.862717Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2304.06364","last_updated":"2023-09-18T14:23:02Z","snapshot_observed_at":"2026-08-07T12:51:50.861720Z","submitted_at":"2023-04-13T09:39:30Z","title":"AGIEval: A Human-Centric Benchmark for Evaluating Foundation Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2304.06364","snapshot_observed_at":"2026-08-15T17:21:06.866133Z","title":"Agieval: A human-centric benchmark for evaluating foundation models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","version":1},"reference_index":70,"source":"pdf_text","source_observed_at":"2026-08-15T17:21:06.866133Z"},"links":{"cited_paper":"/paper/2304.06364","citing_paper":"/paper/2508.13144"},"observation_digest":"sha256:e483fa68d5a35a47b3559d5c80631fee9a499a1b2a327a66df85477658808991","observation_id":"7828047d-0797-4a6d-9b9a-37881801c231","resolution":{"observed_at":"2026-08-15T17:21:06.866133Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2508.13144","last_updated":"2025-08-18T17:56:04Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-16T15:21:53.930509Z","submitted_at":"2025-08-18T17:56:04Z","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation"},"reference_resolution":{"displayed":70,"state_counts":{"malformed_identifier":1,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":52,"verified_exact":1,"verified_fuzzy":16},"total_outbound_references":70},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"thesis":"As of 18 August 2026, this Paper Citation Record lists 70 of 70 outbound references and 5 inbound Pith citation observations for arXiv:2508.13144."}