{"as_of":"2026-08-13T07:12:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:9b104e33d166d87964a9b74e24b5ab4547f3245a6f655a621c0ff27ef840f102","coverage":[{"denominator":29,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":29,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T14:29:54.284637Z","state":"measured"},{"denominator":30,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":30,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-13T06:32:02.005865+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-05-18T18:27:23.076544Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-05-18T18:31:44.524453Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"cited_work":{"arxiv_id":"2411.15320","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2411.15320","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Pplqa: An unsupervised information-theoretic quality metric for comparing generative large language models","venue":null,"work_id":"545edd6b-ba0f-4f25-8cd6-ed65ba9d9010","year":2024},"citing_paper":{"arxiv_id":"2509.05489","last_updated":"2026-04-16T20:13:16Z","snapshot_observed_at":"2026-08-12T18:34:11.831197Z","submitted_at":"2025-09-05T20:39:43Z","title":"Self-Aligned Reward: Towards Effective and Efficient Reasoners","version":2},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-05-18T18:27:23.076544Z"},"links":{"cited_paper":"/paper/2411.15320","citing_paper":"/paper/2509.05489"},"observation_digest":"sha256:142fd14b832965c3c07472679e3ebe2948f3742487b56b34e4f5055351f543a4","observation_id":"59e10ad5-fa82-4eba-8981-577f18095a8d","resolution":{"observed_at":"2026-05-18T18:31:44.527457Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2411.15320/citation-record","integrity":"/paper/2411.15320/integrity","json":"/paper/2411.15320/citation-record.json","paper":"/paper/2411.15320"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.171039Z","title":", \" * write output.state after.block = add.period write newline","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.171039Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:73974be4cce5e3ddee4562422bcab0351ddae7fa38cdd019e43651b3746944ca","observation_id":"f6a8cb7d-5d4d-453d-af05-4bb025f95416","resolution":{"observed_at":"2026-08-12T14:29:54.171039Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.176096Z","title":"write newline","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.176096Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:d38ca72a3848110e0f023c34d72ddb25ba2effd224204d3e340d3e742ec84e61","observation_id":"af84ff0f-652b-464d-af39-d514e287583a","resolution":{"observed_at":"2026-08-12T14:29:54.176096Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.04181","last_updated":"2023-11-04T11:50:13Z","snapshot_observed_at":"2026-07-06T15:39:33.825891Z","submitted_at":"2023-06-07T06:29:58Z","title":"Benchmarking Foundation Models with Language-Model-as-an-Examiner","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.04181","snapshot_observed_at":"2026-08-12T14:29:54.180727Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.180727Z"},"links":{"cited_paper":"/paper/2306.04181","citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:aa38efe1d3ce29383ae5a8262298acad654f930cac10b95a4d63a85d12be20d4","observation_id":"4f807f6a-add5-4a53-a64a-1e10b2e9877c","resolution":{"observed_at":"2026-08-12T14:29:54.180727Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.600742Z","title":null,"venue":null,"work_id":"1ca01067-52ca-40d3-b450-b75b4b8a39ca","year":2005},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.185421Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:44ba15f8371a08009cade8aa500d4c28d5b5d81dab6ae84f731b365391e51ec5","observation_id":"f97974cb-c8d3-44c4-a663-cad246c47a33","resolution":{"observed_at":"2026-08-12T14:29:54.604747Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.588139Z","title":"J.; and Jurman, G","venue":null,"work_id":"366c2fb2-fb5d-4942-94b0-a555f9a61032","year":2021},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.189712Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:810d3d027678d99690c0fab69bd8d25ec79d58f42996b0bb13dc8d2f529de26b","observation_id":"92ede92c-4fd9-40fe-afec-76154bfe11be","resolution":{"observed_at":"2026-08-12T14:29:54.591881Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.576461Z","title":null,"venue":null,"work_id":"80d59186-488c-4817-a8f5-da964b9bebde","year":2024},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.193802Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:33b10e8f131aa2bb80b2c5cf31459597fcd36a8e9bfc5bb6e16300e992fad5a3","observation_id":"95851375-d731-4b6a-9afb-76c012183f61","resolution":{"observed_at":"2026-08-12T14:29:54.580303Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.564793Z","title":null,"venue":null,"work_id":"9d3bd986-2b2e-4f40-b923-a3921611d78c","year":2020},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.197870Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:69ea791415367504ae6ccee3a602022eabed6c70ab63e70a2121648e7df4dbdf","observation_id":"7f7695b5-5bea-4e40-a80e-11962a72be5f","resolution":{"observed_at":"2026-08-12T14:29:54.569592Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.553574Z","title":null,"venue":null,"work_id":"4ce1e3f7-f522-4436-b30a-4db95bb40d87","year":2024},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.201945Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:48c28b83a8d414180f9efba3125526cd12f4d8cd9493fb0c3019ca6b608c1b93","observation_id":"9862311d-7274-436e-a049-4d33d266bacb","resolution":{"observed_at":"2026-08-12T14:29:54.557458Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.541665Z","title":null,"venue":null,"work_id":"57148762-4709-4ca6-aaef-f58ee9e57765","year":2023},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.206207Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:8ad78bd442c6472ebe6d1bb8187dc0dacd77eaccbb946e96b0c046b6cffe90be","observation_id":"5736b260-f1f9-4fb1-a2ab-193f4afdaf42","resolution":{"observed_at":"2026-08-12T14:29:54.545583Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2302.04166","last_updated":"2023-02-13T16:19:05Z","snapshot_observed_at":"2026-08-05T08:58:00.258106Z","submitted_at":"2023-02-08T16:17:29Z","title":"GPTScore: Evaluate as You Desire","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.04166","snapshot_observed_at":"2026-08-12T14:29:54.210042Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.210042Z"},"links":{"cited_paper":"/paper/2302.04166","citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:b34fea841fa8b69769b0ecaa2d9ce2cb694ed2ba1d5a4b41d112d1e1f935ce6c","observation_id":"eca368e3-d7ab-4f54-ace5-abb1cacc6d0b","resolution":{"observed_at":"2026-08-12T14:29:54.210042Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.214139Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.214139Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:7e0389f0be80bae21024bc0dd35b03e7b94f17d54fe9535f9b06d5937005b34b","observation_id":"51f47c82-cf10-4bf2-98dd-215352389f4d","resolution":{"observed_at":"2026-08-12T14:29:54.214139Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2212.04037","last_updated":"2024-09-12T19:54:37Z","snapshot_observed_at":"2026-08-07T11:18:39.827875Z","submitted_at":"2022-12-08T02:21:47Z","title":"Demystifying Prompts in Language Models via Perplexity Estimation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2212.04037","snapshot_observed_at":"2026-08-12T14:29:54.218118Z","title":"A.; and Zettlemoyer, L","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.218118Z"},"links":{"cited_paper":"/paper/2212.04037","citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:708c423b552cd3b23bc293e53a2280e50c153def5531d87f2bf72f02395f2cc9","observation_id":"6629d659-0bc5-404a-83cd-4b4a290c9877","resolution":{"observed_at":"2026-08-12T14:29:54.218118Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.08446","last_updated":"2025-02-11T18:59:26Z","snapshot_observed_at":"2026-08-12T23:44:18.408965Z","submitted_at":"2024-06-12T17:37:09Z","title":"OLMES: A Standard for Language Model Evaluations","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.08446","snapshot_observed_at":"2026-08-12T14:29:54.222181Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.222181Z"},"links":{"cited_paper":"/paper/2406.08446","citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:d816d7ec9a9511b0e3c57705300082062725d7ffa8b737f435f3339cde5b2efa","observation_id":"c6bcbf22-ed82-44d4-a177-9ab5992b46aa","resolution":{"observed_at":"2026-08-12T14:29:54.222181Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T14:29:54.226325Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.226325Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:c59c0a372cd025dd58977dfff257af121876928060c1164486ef9c067f8f1351","observation_id":"45e17965-2ffe-431c-bd3b-2ba97c179f1d","resolution":{"observed_at":"2026-08-12T14:29:54.226325Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.521501Z","title":null,"venue":null,"work_id":"2d92ce3b-73d7-4332-a9ed-076714c0c481","year":2004},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.230138Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:41df3785664e34bf894a5dfc136e9286a96ec553b23bd5d66085532d60efb201","observation_id":"d0ec3e1e-495e-4972-8c14-7cd3f979194f","resolution":{"observed_at":"2026-08-12T14:29:54.526086Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2305.13711","last_updated":"2023-05-23T05:57:09Z","snapshot_observed_at":"2026-08-11T14:50:51.066890Z","submitted_at":"2023-05-23T05:57:09Z","title":"LLM-Eval: Unified Multi-Dimensional Automatic Evaluation for Open-Domain Conversations with Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.13711","snapshot_observed_at":"2026-08-12T14:29:54.233656Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.233656Z"},"links":{"cited_paper":"/paper/2305.13711","citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:ef3bda04bfbc243af10d19570f3d382b919813ba2ec9121bcd384af88b0e6293","observation_id":"0b709059-0e66-46cb-adbc-02615707342e","resolution":{"observed_at":"2026-08-12T14:29:54.233656Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.510292Z","title":null,"venue":null,"work_id":"1e2e1229-6496-40ea-90e5-6097b31d516e","year":2023},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.237459Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:f7e0cd445a23924057c50def31382e9a36cc52740fca218096be428aef45a870","observation_id":"e2fbde52-920a-424f-b224-4d129ac0a1a6","resolution":{"observed_at":"2026-08-12T14:29:54.514068Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-07T07:30:12.213965Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-12T14:29:54.241101Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.241101Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:618d4b3350896ae696f5473610b5db8fb3d1f4887d6a798ef9e704db0efc1bee","observation_id":"c37fd807-f80c-4f3c-a7b2-a4f2510030dd","resolution":{"observed_at":"2026-08-12T14:29:54.241101Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.498460Z","title":null,"venue":null,"work_id":"e92ec164-c1a5-4e21-a8f5-0789d64de23a","year":2002},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.245111Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:442faefc24fc3d916f59830fbcd10df4d2e389ce62830996b8c89bf651bc4293","observation_id":"b80bfecf-3dca-4998-bdb8-1e6f73568866","resolution":{"observed_at":"2026-08-12T14:29:54.502295Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2010.16061","last_updated":"2020-10-11T02:15:11Z","snapshot_observed_at":"2026-08-04T20:09:58.097029Z","submitted_at":"2020-10-11T02:15:11Z","title":"Evaluation: from precision, recall and F-measure to ROC, informedness, markedness and correlation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2010.16061","snapshot_observed_at":"2026-08-12T14:29:54.248917Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.248917Z"},"links":{"cited_paper":"/paper/2010.16061","citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:4ebba3afbf3b6111c634408d3a61ef0cfb750144e7025f31b3991b2ea25c0243","observation_id":"68deed28-a79c-4e4a-9e8e-b625b5523a9a","resolution":{"observed_at":"2026-08-12T14:29:54.248917Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2206.04615","last_updated":"2023-06-12T17:51:15Z","snapshot_observed_at":"2026-07-06T13:19:12.109592Z","submitted_at":"2022-06-09T17:05:34Z","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2206.04615","snapshot_observed_at":"2026-08-12T14:29:54.253765Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.253765Z"},"links":{"cited_paper":"/paper/2206.04615","citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:981ee79ebdc151f5763599556514bb5b860246f0d5428b47df50a37acfc283dc","observation_id":"1c4e7523-582a-4ae9-a987-67840539f7b4","resolution":{"observed_at":"2026-08-12T14:29:54.253765Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.487570Z","title":"L.; and Parikh, D","venue":null,"work_id":"bc33fcf7-1bb1-421a-91bf-f8a7c86d1ea7","year":2015},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.257663Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:fdf4b43a71f76fbfcbd78c16f2f2d443437c23e5c33cec77bd949257f3ec0eec","observation_id":"afc954c1-e2ff-4f77-9be5-60d365c448c6","resolution":{"observed_at":"2026-08-12T14:29:54.491310Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.475570Z","title":null,"venue":null,"work_id":"8e4849ec-93cd-4e92-8c5f-53193e56f77d","year":2019},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.261221Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:db4571ccb09dc90aa3a3486ea782083b959dabe26d8e284913edd53c8c549500","observation_id":"ec311917-7048-4f14-a72a-bea9dfee9684","resolution":{"observed_at":"2026-08-12T14:29:54.479863Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.463013Z","title":null,"venue":null,"work_id":"7bdd8d87-cd3c-4744-92ac-17e062a522d3","year":2018},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.264645Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:b53f04065dfe3c3bad47feaa58374ff5bad1ac6f7dbf313f173457cbfe0ae649","observation_id":"fd272bb9-6ebb-4a7b-aa4e-7e87dcc15d8e","resolution":{"observed_at":"2026-08-12T14:29:54.467074Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:29:54.449331Z","title":null,"venue":null,"work_id":"12ce7be3-5daa-456e-b789-3ac7b273d895","year":2024},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.268688Z"},"links":{"citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:caeb480122e54bd541e6cf40cb3671bcb2ca2b1d29033dae23b352614abee186","observation_id":"068c3815-3ce9-4658-9630-25aa6eacc72c","resolution":{"observed_at":"2026-08-12T14:29:54.454739Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2202.13169","last_updated":"2022-05-04T16:08:31Z","snapshot_observed_at":"2026-08-06T20:27:50.671898Z","submitted_at":"2022-02-26T15:53:55Z","title":"A Systematic Evaluation of Large Language Models of Code","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2202.13169","snapshot_observed_at":"2026-08-12T14:29:54.272514Z","title":"F.; Alon, U.; Neubig, G.; and Hellendoorn, V","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.272514Z"},"links":{"cited_paper":"/paper/2202.13169","citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:86b46c2ff095daa9e3c81055e8bfe219ae04e69d85861e2823f079a6da26fcf3","observation_id":"3aa9fed1-7440-45ca-b0bd-3b2d6da62464","resolution":{"observed_at":"2026-08-12T14:29:54.272514Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1902.03570","last_updated":"2019-02-10T10:34:54Z","snapshot_observed_at":"2026-08-01T23:01:42.147835Z","submitted_at":"2019-02-10T10:34:54Z","title":"EvalAI: Towards Better Evaluation Systems for AI Agents","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1902.03570","snapshot_observed_at":"2026-08-12T14:29:54.276421Z","title":"B.; Lee, S.; and Batra, D","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.276421Z"},"links":{"cited_paper":"/paper/1902.03570","citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:fe4d08d19e561c0d9244de84e87a626cb96034b7415eb233858728eecc798760","observation_id":"05efc8a0-510d-4f6a-b0ff-b90056466895","resolution":{"observed_at":"2026-08-12T14:29:54.276421Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1904.09675","last_updated":"2020-02-24T18:59:28Z","snapshot_observed_at":"2026-07-29T15:42:51.774083Z","submitted_at":"2019-04-21T23:08:53Z","title":"BERTScore: Evaluating Text Generation with BERT","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1904.09675","snapshot_observed_at":"2026-08-12T14:29:54.280278Z","title":"Q.; and Artzi, Y","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.280278Z"},"links":{"cited_paper":"/paper/1904.09675","citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:f6be1c3af79e207f642b928bba510ff2977e3ce03e057da1f11afdce95ba8225","observation_id":"333d1227-68fb-4d2e-8c22-969d030b4255","resolution":{"observed_at":"2026-08-12T14:29:54.280278Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.05685","last_updated":"2023-12-24T02:01:34Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-06-09T05:55:52Z","title":"Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.05685","snapshot_observed_at":"2026-08-12T14:29:54.284637Z","title":"P.; Zhang, H.; Gonzalez, J","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","version":1},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-12T14:29:54.284637Z"},"links":{"cited_paper":"/paper/2306.05685","citing_paper":"/paper/2411.15320"},"observation_digest":"sha256:7153d6309dbbb4edd2011e65e3331d3c27cd2cf7432ffeaed943304069770351","observation_id":"f21dfe25-205d-4a58-be6d-cef4100443c4","resolution":{"observed_at":"2026-08-12T14:29:54.284637Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2411.15320","last_updated":"2024-11-22T19:28:06Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-12T14:23:39.254918Z","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models"},"reference_resolution":{"displayed":29,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":27,"verified_exact":0,"verified_fuzzy":2},"total_outbound_references":29},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"thesis":"As of 13 August 2026, this Paper Citation Record lists 29 of 29 outbound references and 1 inbound Pith citation observation for arXiv:2411.15320."}