{"as_of":"2026-08-16T06:40:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:df912d43880d5063da20054735d39f8d3649c8c365d0a1d3a12aee8bdf82aaf3","coverage":[{"denominator":41,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":41,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-10T23:01:05.175338Z","state":"measured"},{"denominator":43,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":43,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-16T06:30:59.297886+00:00","state":"measured"},{"denominator":2,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":2,"source":"paper_references, paper_reference_links","source_observed_at":"2026-06-27T10:13:22.927828Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-03T10:07:56.328267Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"cited_work":{"arxiv_id":"2501.00257","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.00257","snapshot_observed_at":"2026-07-03T10:07:56.328267Z","title":"0.0-beta , author=","venue":null,"work_id":"0d8aa165-664f-402c-a818-31f36083c32e","year":2024},"citing_paper":{"arxiv_id":"2605.15486","last_updated":"2026-05-15T00:04:38Z","snapshot_observed_at":"2026-08-16T01:45:08.769771Z","submitted_at":"2026-05-15T00:04:38Z","title":"Hybrid LLM-based Intelligent Framework for Robot Task Scheduling","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-05-19T16:21:19.722476Z"},"links":{"cited_paper":"/paper/2501.00257","citing_paper":"/paper/2605.15486"},"observation_digest":"sha256:c403a9ab9c8b466102f5fff1e2ef3568598670b65afe6b2da60c4db11997a7cb","observation_id":"42c8eb86-374a-40d3-8b79-4964b2a39105","resolution":{"observed_at":"2026-05-19T16:22:39.541189Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"cited_work":{"arxiv_id":"2501.00257","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.00257","snapshot_observed_at":"2026-07-03T10:07:56.328267Z","title":"0.0-beta , author=","venue":null,"work_id":"0d8aa165-664f-402c-a818-31f36083c32e","year":2024},"citing_paper":{"arxiv_id":"2606.11643","last_updated":"2026-06-10T04:07:41Z","snapshot_observed_at":"2026-08-13T04:42:19.291169Z","submitted_at":"2026-06-10T04:07:41Z","title":"Improving Cross-Format Robustness in Language Models with Multi-Format Training","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-06-27T10:13:22.927828Z"},"links":{"cited_paper":"/paper/2501.00257","citing_paper":"/paper/2606.11643"},"observation_digest":"sha256:525635de5a640b50a8db50d5fb9674faec317c3e5e12d6a248fecb96d7fc0608","observation_id":"0478b8ac-07c5-4e89-9319-ea126317238f","resolution":{"observed_at":"2026-07-03T10:07:56.329771Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2501.00257/citation-record","integrity":"/paper/2501.00257/integrity","json":"/paper/2501.00257/citation-record.json","paper":"/paper/2501.00257"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:06.143955Z","title":"Easy problems that llms get wrong","venue":null,"work_id":"aec95576-fe8a-4e30-b8fe-fce73af06ef5","year":2024},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:04.991198Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:f546fe9217ba94de0f9533019e194db066b9ad67a668b003f884c46bc429e9c9","observation_id":"88aeec0e-04dc-43dc-a02e-2aa1fe330a44","resolution":{"observed_at":"2026-08-10T23:01:06.147951Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:06.131533Z","title":"Wider and deeper llm networks are fairer llm evaluators","venue":null,"work_id":"4de1c197-4a0c-4283-a177-9df9937bc6b7","year":2023},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:04.995596Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:15a1397b19231b6ba05d01cd36363e6a430c7524328b61b3427d1205dfee9902","observation_id":"98cc8109-6d03-4cbf-92e9-25e880f9240e","resolution":{"observed_at":"2026-08-10T23:01:06.135629Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:04.999522Z","title":"Likelihood-based mitigation of evaluation bias in large language models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:04.999522Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:598d4eeb6120bb0c9ccd983cfa85e346cefb4377aa4682678b318dd1c9fb13de","observation_id":"c38bcac9-4dd5-4ac1-b8c2-da444060f07b","resolution":{"observed_at":"2026-08-10T23:01:04.999522Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.01100","last_updated":"2025-03-31T20:37:34Z","snapshot_observed_at":"2026-08-14T01:08:51.087077Z","submitted_at":"2024-07-01T09:06:57Z","title":"Eliminating Position Bias of Language Models: A Mechanistic Approach","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.01100","snapshot_observed_at":"2026-08-10T23:01:05.004248Z","title":"Eliminating position bias of language models: A mechanistic approach,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.004248Z"},"links":{"cited_paper":"/paper/2407.01100","citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:628f6826015f75db96a9c21a094281c477b432beeaf51dd04862fa192d128aaf","observation_id":"4e4ffac1-4a65-4adb-92fa-df9d93044a48","resolution":{"observed_at":"2026-08-10T23:01:05.004248Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.14275","last_updated":"2024-03-21T10:31:11Z","snapshot_observed_at":"2026-08-13T00:48:48.283262Z","submitted_at":"2024-03-21T10:31:11Z","title":"Is Reference Necessary in the Evaluation of NLG Systems? When and Where?","version":1},"cited_work":{"arxiv_id":"2403.14275","doi":null,"metadata_source":"pith","pith_arxiv_id":"2403.14275","snapshot_observed_at":"2026-08-10T23:01:05.593508Z","title":"Is Reference Necessary in the Evaluation of NLG Systems? When and Where?","venue":"cs.CL","work_id":"d0f1ee98-ecd5-4ce7-b9a5-fbfed08771bd","year":2024},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.009442Z"},"links":{"cited_paper":"/paper/2403.14275","citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:7abd877c21590011ed794eb1f8940d8a22fecdae55ab0af452ffa482a5fab10e","observation_id":"919b7864-f95b-4f15-8e20-127e29523506","resolution":{"observed_at":"2026-08-10T23:01:05.708641Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2307.03025","last_updated":"2023-11-12T06:25:32Z","snapshot_observed_at":"2026-08-13T20:16:44.835319Z","submitted_at":"2023-07-06T14:42:01Z","title":"Style Over Substance: Evaluation Biases for Large Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.03025","snapshot_observed_at":"2026-08-10T23:01:05.014038Z","title":"Style over substance: Evaluation biases for large language models,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.014038Z"},"links":{"cited_paper":"/paper/2307.03025","citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:5cfd7a6e683b2dba37ac7427093134043a773cb42ecbb4e6554ad63febcb1fcc","observation_id":"71448988-0eea-4d2b-af94-4d6b91ca0f03","resolution":{"observed_at":"2026-08-10T23:01:05.014038Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.01399","last_updated":"2025-01-05T22:23:35Z","snapshot_observed_at":"2026-08-14T06:56:40.072806Z","submitted_at":"2024-04-01T18:10:05Z","title":"Developing Safe and Responsible Large Language Model : Can We Balance Bias Reduction and Language Understanding in Large Language Models?","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.01399","snapshot_observed_at":"2026-08-10T23:01:05.019358Z","title":"Developing safe and responsible large language models–a comprehensive framework,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.019358Z"},"links":{"cited_paper":"/paper/2404.01399","citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:31491f1f91eb5b673fb433b314ab7354e2adbee994b89b0b6dd11515d5d77fb2","observation_id":"b1994f4a-8592-4206-8661-7595b712b042","resolution":{"observed_at":"2026-08-10T23:01:05.019358Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.01937","last_updated":"2023-05-03T07:28:50Z","snapshot_observed_at":"2026-08-13T11:50:17.843572Z","submitted_at":"2023-05-03T07:28:50Z","title":"Can Large Language Models Be an Alternative to Human Evaluations?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.01937","snapshot_observed_at":"2026-08-10T23:01:05.024146Z","title":"Can large language models be an alternative to human evaluations?","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.024146Z"},"links":{"cited_paper":"/paper/2305.01937","citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:d59d26d97db484a588ccc24704a141b177d2cba8048926d4726add079d67b470","observation_id":"30aa11da-13e1-4ed8-ab8d-c1c7a56d4f0e","resolution":{"observed_at":"2026-08-10T23:01:05.024146Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.18711","last_updated":"2024-12-05T04:01:28Z","snapshot_observed_at":"2026-08-12T23:55:35.180473Z","submitted_at":"2024-05-29T02:44:12Z","title":"Calibrating Reasoning in Language Models with Internal Consistency","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.18711","snapshot_observed_at":"2026-08-10T23:01:05.028500Z","title":"Calibrating reasoning in language models with internal consistency,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.028500Z"},"links":{"cited_paper":"/paper/2405.18711","citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:46d09f61fe66671efe3f7f282ff76b370f614d9d167970afbf6d90a6ca80e2e5","observation_id":"5181b783-d8b3-4886-b614-6d16f5062f3b","resolution":{"observed_at":"2026-08-10T23:01:05.028500Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:06.116290Z","title":"Investigating automatic scoring and feedback using large language models,","venue":null,"work_id":"19b63b7d-f462-4f0a-9c22-5c4ffe7a7ee3","year":2024},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.032816Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:1e9e8e0492f2f4a52375805b776d2748cfb415d467de11dd22ecc77b00181706","observation_id":"dc3a4243-4793-4e0f-adb9-efc3d43ec620","resolution":{"observed_at":"2026-08-10T23:01:06.120911Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1905.00537","last_updated":"2020-02-13T00:28:00Z","snapshot_observed_at":"2026-08-12T12:17:26.135577Z","submitted_at":"2019-05-02T00:41:50Z","title":"SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1905.00537","snapshot_observed_at":"2026-08-10T23:01:05.036834Z","title":"Superglue: A stickier benchmark for general-purpose language understanding systems,","venue":null,"work_id":null,"year":1905},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.036834Z"},"links":{"cited_paper":"/paper/1905.00537","citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:c0983f4ddf92ff1650a51b3695eb461aeffd5b38a1068041bf03dd062b427bc8","observation_id":"ad6b61c0-c0bb-4514-ba2b-36edc90e2bc3","resolution":{"observed_at":"2026-08-10T23:01:05.036834Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2206.04615","last_updated":"2023-06-12T17:51:15Z","snapshot_observed_at":"2026-07-06T13:19:12.109592Z","submitted_at":"2022-06-09T17:05:34Z","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2206.04615","snapshot_observed_at":"2026-08-10T23:01:05.041156Z","title":"Beyond the imitation game: Quantifying and extrapolating the capabilities of language models,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.041156Z"},"links":{"cited_paper":"/paper/2206.04615","citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:0ec597afb1ec5b0a102be1fa728b2029adf345623adeaa8bf7b5f920827ce5aa","observation_id":"d537cba4-f00a-4fdc-8a62-f1585623981b","resolution":{"observed_at":"2026-08-10T23:01:05.041156Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.11966","last_updated":"2024-06-26T07:16:42Z","snapshot_observed_at":"2026-08-13T00:16:49.895003Z","submitted_at":"2024-05-20T11:47:13Z","title":"Multiple-Choice Questions are Efficient and Robust LLM Evaluators","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.11966","snapshot_observed_at":"2026-08-10T23:01:05.046176Z","title":"Multiple-choice questions are efficient and robust llm evaluators,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.046176Z"},"links":{"cited_paper":"/paper/2405.11966","citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:83a6421858036a484c31f12b368d5f68c99b9caf73c214b3919692a8d9bf3f85","observation_id":"0091f0e4-15eb-4ed8-8a54-b22ce563682f","resolution":{"observed_at":"2026-08-10T23:01:05.046176Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1908.10084","last_updated":"2019-08-27T08:50:17Z","snapshot_observed_at":"2026-08-14T05:02:11.716316Z","submitted_at":"2019-08-27T08:50:17Z","title":"Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1908.10084","snapshot_observed_at":"2026-08-10T23:01:05.050418Z","title":"Sentence-bert: Sentence embeddings using siamese bert- networks,","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.050418Z"},"links":{"cited_paper":"/paper/1908.10084","citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:010f9754fac7e5254a5925cc023f46e43c75cfcb7bed7f353fca502e95b58441","observation_id":"95a6f582-434a-4cc3-a5f1-a2e516786e7c","resolution":{"observed_at":"2026-08-10T23:01:05.050418Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.04948","last_updated":"2024-08-09T09:07:48Z","snapshot_observed_at":"2026-08-12T23:06:07.176815Z","submitted_at":"2024-08-09T09:07:48Z","title":"HybridRAG: Integrating Knowledge Graphs and Vector Retrieval Augmented Generation for Efficient Information Extraction","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.04948","snapshot_observed_at":"2026-08-10T23:01:05.054412Z","title":"Hybridrag: Integrating knowledge graphs and vector retrieval augmented generation for efficient information extraction,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.054412Z"},"links":{"cited_paper":"/paper/2408.04948","citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:08e206c48a7cf2b049b9b1ea56b3bc49cd241c52dfefb170bf0334d178b23602","observation_id":"a6f32267-84db-447c-9f6a-117e6592ba28","resolution":{"observed_at":"2026-08-10T23:01:05.054412Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:06.103261Z","title":"Evaluations are critical for understanding the capabilities of large language models (llms),","venue":null,"work_id":"207e5aaa-6692-408c-a64e-323bf6e8eea3","year":2024},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.058834Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:842bb8f0ab4aec4fe958ff202993a9e566bec5a459aa6d354ccb4885c09f1dda","observation_id":"02f17650-ef9a-4586-8f96-c346e3425b57","resolution":{"observed_at":"2026-08-10T23:01:06.107389Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:06.091719Z","title":"Exploring bias and prediction metrics to characterise the fairness of machine learning for equity-centered public health decision-making: A narrative review,","venue":null,"work_id":"97112473-d86e-489c-8afe-3f9dd42d8d57","year":2024},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.063244Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:2cc989eb4f5ba0d270f69886222da1bff3ec54651d9fab9ae44a5c9303898287","observation_id":"a4bdfffe-f263-4c9a-9483-dfe02acdf823","resolution":{"observed_at":"2026-08-10T23:01:06.095920Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:06.080096Z","title":"Fake news detection: Comparative evaluation of bert-like models and large language models with generative ai-annotated data,","venue":null,"work_id":"8d5ab7a5-cc5a-49cc-9663-39776e957c53","year":2024},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.067062Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:098bdbb62f55fe85b207b787bbcf4d31957e5616c35827c87a7b8b21c58df98c","observation_id":"160e2bd3-fa5b-4f0f-969c-8db522327a6a","resolution":{"observed_at":"2026-08-10T23:01:06.084624Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1904.03310","last_updated":"2019-04-05T22:36:12Z","snapshot_observed_at":"2026-08-14T16:50:52.634400Z","submitted_at":"2019-04-05T22:36:12Z","title":"Gender Bias in Contextualized Word Embeddings","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1904.03310","snapshot_observed_at":"2026-08-10T23:01:05.070741Z","title":"Gender bias in contextualized word embeddings,","venue":null,"work_id":null,"year":1904},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.070741Z"},"links":{"cited_paper":"/paper/1904.03310","citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:700cd1cbcba35eb23977fc546de070343cdab189a029e40c3267a18183db268f","observation_id":"28f090c3-1cf8-4b2d-b148-b1467cdaf159","resolution":{"observed_at":"2026-08-10T23:01:05.070741Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2412.17052","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.435800Z","title":"Vilbias: A framework for bias detection using linguistic and visual cues,","venue":null,"work_id":"7e586f4d-ac96-443e-9631-7071bd43c478","year":2024},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.074678Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:ec92fd270077c59e2b131d7971f542df48f2776169e87f3c170dd20972c21c13","observation_id":"f7c52f11-703d-46f4-a9d4-c3cc1ed79500","resolution":{"observed_at":"2026-08-10T23:01:05.445888Z","resolver_source":"raw_fallback","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:06.067591Z","title":"Usability maturity model: Human centredness scale,","venue":null,"work_id":"6aa7f28e-1aca-44f8-9c79-ecf2175c6739","year":1998},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.079004Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:3766e713466629dd92083928ff99ac353bfa647a2f505fb02ba3b9bc2906241a","observation_id":"d062cd46-0c09-44ba-9e1f-b33e2882fbcf","resolution":{"observed_at":"2026-08-10T23:01:06.072497Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:06.054491Z","title":"Cohen, Statistical Power Analysis for the Behavioral Sciences","venue":null,"work_id":"550d4601-d0a7-49ab-96e1-7204531815e7","year":1988},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.083143Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:b535617d47e842becd2baab77a6ca0536ebeeb2413736e0e0dc74dc66b0589ef","observation_id":"3cf6f5e5-8b38-41e1-97f7-0d1bff4a7367","resolution":{"observed_at":"2026-08-10T23:01:06.058887Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:06.037797Z","title":"Cohen, Statistical Power Analysis for the Behavioral Sciences","venue":null,"work_id":"9fdccec6-de6e-4ef8-9f0d-49ac60633219","year":1969},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.087412Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:4c5b847ce3ebf881705b18c35b3c2d1ed31d3c8d5abde7dd2a0b750c8ce1c5cb","observation_id":"d8523b20-6e6a-42f0-a7d6-ad7a2fe4a815","resolution":{"observed_at":"2026-08-10T23:01:06.046042Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:06.019976Z","title":"New effect size rules of thumb,","venue":null,"work_id":"6b5cf1db-1e1c-4132-88dd-1190c0cf9974","year":2009},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.092038Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:34450e238bddfc5fc1d22018f9de655fae9e330f1ecf73ad6d9e101c906955c7","observation_id":"3326e2ab-d9a2-43eb-b656-12ac69f29f96","resolution":{"observed_at":"2026-08-10T23:01:06.024271Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:06.007928Z","title":"Calculating and reporting effect sizes to facilitate cumulative science: A practical primer for t-tests and anovas,","venue":null,"work_id":"a86b1ec1-b5fa-43a0-a069-02044b0e1ee6","year":1988},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.096178Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:0eaa6c13bf1471156dced42b01d16d5c5aa499803c80f59bee88af87f62b29d5","observation_id":"ada8d743-b219-42e6-a95d-f0288407b618","resolution":{"observed_at":"2026-08-10T23:01:06.012329Z","resolver_source":"raw_fallback","status":"malformed_identifier"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.994478Z","title":null,"venue":null,"work_id":"6e63b7b5-506a-495c-a945-73abd1b6bb51","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.101515Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:2043a590efefbb780b55829cafd27032d73514a0c02a17d9d0c997605057a1cb","observation_id":"f7eeab6a-e473-45ac-9484-ca0408855362","resolution":{"observed_at":"2026-08-10T23:01:05.998912Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.981941Z","title":null,"venue":null,"work_id":"773341e4-703a-4165-bd10-14998f3b6801","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.105726Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:e4c400d9e919fd8070d4b6ee155319d709360d726f7e6fb4cf2ed2c611b4bbe4","observation_id":"97674471-872d-4a43-8fd7-cfc1a24921b5","resolution":{"observed_at":"2026-08-10T23:01:05.985984Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.970199Z","title":"Human-Eval","venue":null,"work_id":"f5da5532-7f70-4cdf-942c-d8852075292c","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.110234Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:6b1972ac2f9e29a39a7c18bccc3c7bc01a2a2f2079e8e3e5f6eb6f86dbec4101","observation_id":"3e13059f-c86d-4c40-98c1-c5521b4cae4b","resolution":{"observed_at":"2026-08-10T23:01:05.974031Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.956410Z","title":"• Length of the Bar: The longer the bar, the greater the effect size, meaning the model’s performance deviates more significantly from the EQUATOR evaluator","venue":null,"work_id":"61d6657d-392d-42f3-9722-de892edebde1","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.114552Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:b17b52558fa7e42b137c21b823423efcb627872978bc1230d5b620df6fa5befb","observation_id":"578eca70-1d96-4042-8efe-da6bf14b93dc","resolution":{"observed_at":"2026-08-10T23:01:05.961202Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.944269Z","title":"– Higher values mean a larger difference between the mean scores, standardized by the pooled variability","venue":null,"work_id":"71ecca99-1c55-49d6-836e-ac13420edfc3","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.118676Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:d7a030a0c90445b68788cdcd09913b89b12906b6771cda1520835ba3be5f1577","observation_id":"e0b85857-014a-4e37-a229-9e8124b00c8f","resolution":{"observed_at":"2026-08-10T23:01:05.948640Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.933115Z","title":null,"venue":null,"work_id":"4d8f9692-062e-40fd-9ade-8e772b7e3105","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.122512Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:2575ad6b03087c8a041febd3256be2a9c4c1b9cc35c2a1ea5205eec4b2c2a707","observation_id":"f43caacc-ba0d-42e1-9dca-4c2f796bcbfc","resolution":{"observed_at":"2026-08-10T23:01:05.936795Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.921548Z","title":null,"venue":null,"work_id":"13665044-3b39-4882-9816-9aba5117c485","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.128120Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:3ea10dd506196e0ddcc8c4939f6f19d42822675560c7b0c95a0114982783cf30","observation_id":"bab3a378-174a-4712-bd00-2973322a4400","resolution":{"observed_at":"2026-08-10T23:01:05.925327Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.910320Z","title":null,"venue":null,"work_id":"fa54cf23-61b1-43e8-9833-321011fe4b34","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.132175Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:837cb7952807080987958617c9481aa5efc654ec95881dfe6b1e0a3021d6caa9","observation_id":"47203ee1-7968-4860-910c-561dd076de8a","resolution":{"observed_at":"2026-08-10T23:01:05.914179Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.897775Z","title":null,"venue":null,"work_id":"db3d1b63-f8bf-4ba7-a99f-2cf29b717b92","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.136389Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:7823fafd7e38909e0e641a06322a05903c45790dc3ce665a402192a24222d699","observation_id":"eac7ffff-97f9-4208-ab5e-c19e3cafc5be","resolution":{"observed_at":"2026-08-10T23:01:05.901853Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.884948Z","title":"Larger effects require fewer questions","venue":null,"work_id":"11342359-4f1e-49ab-acc3-7d5712312ed6","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.143033Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:25f80b618acb102ae4e62d56da21ea4a7beb463e040735ada7cbd50098d6d81c","observation_id":"a9548bf7-494d-44ed-a4e9-d6151f2ecb7e","resolution":{"observed_at":"2026-08-10T23:01:05.889126Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.872336Z","title":"• These models are relatively aligned with the EQUATOR evaluator, suggesting they produce similar results","venue":null,"work_id":"c01a0d1e-0f3f-46e0-a204-248b5b32bf9f","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.149377Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:b45a28ecf582ca61c4ac8dc3eba50be0618695792cb5446c99930d3d93ac4469","observation_id":"f86df44c-5829-41a8-81d6-0b05a06f1a07","resolution":{"observed_at":"2026-08-10T23:01:05.876754Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.859017Z","title":"• These models perform significantly worse under the given evaluation, as indicated by large positive Cohen’s d values","venue":null,"work_id":"f85c4f15-73e9-4d2f-b1b8-dc35e2355889","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.154981Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:6298405063fd35cd32095b21d8987ee6a6df4d7c2b3fd9028e8a35d66d0a31ff","observation_id":"e7d2daf7-0ddb-4958-b39e-b523e8786954","resolution":{"observed_at":"2026-08-10T23:01:05.863436Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.846673Z","title":"• A wide range of Cohen’s d values (e.g., from˜0.3 to >2.0) indicates substantial variability in model performance","venue":null,"work_id":"98fb3438-137b-4faa-a09e-658f29c7973b","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.160027Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:88d367f7032097a844fba51163c29be8be718eeac886c69e91a245de80eecdd5","observation_id":"0785b479-33f6-4e8d-b79f-7a8c26d733ff","resolution":{"observed_at":"2026-08-10T23:01:05.850675Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.832452Z","title":null,"venue":null,"work_id":"d8e2f3ac-54f5-41e3-a585-a1c2e4fb8ce2","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.165784Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:29970884c7e3d63d63936e77f596c9c43109575c2e2c06426b77374dd7b1cfd1","observation_id":"7b4aa07f-5509-4225-8e08-401831fa7131","resolution":{"observed_at":"2026-08-10T23:01:05.836770Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.816212Z","title":null,"venue":null,"work_id":"908a5a3c-4235-4f5b-a70a-1b95d530168c","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.170926Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:e48208df4c698dd24b652b35cf0ca19a5563f9da11dfe8168cca6eecced1049a","observation_id":"acd28fd1-4a70-420b-9280-b816ee0352ff","resolution":{"observed_at":"2026-08-10T23:01:05.819994Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T23:01:05.803090Z","title":null,"venue":null,"work_id":"94a7c6b9-edb5-4d7f-9354-23332fcc8ccc","year":null},"citing_paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-10T23:01:05.175338Z"},"links":{"citing_paper":"/paper/2501.00257"},"observation_digest":"sha256:14f1c64e86807b8b94516a4e45322a84549968e63a647ade5cc9d109d1ee8214","observation_id":"d9836045-3ec6-4a4c-8783-ccf30441411d","resolution":{"observed_at":"2026-08-10T23:01:05.807268Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2501.00257","last_updated":"2024-12-31T03:56:17Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-14T08:35:19.921704Z","submitted_at":"2024-12-31T03:56:17Z","title":"EQUATOR: A Deterministic Framework for Evaluating LLM Reasoning with Open-Ended Questions. # v1.0.0-beta"},"reference_resolution":{"displayed":41,"state_counts":{"malformed_identifier":1,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":21,"verified_exact":2,"verified_fuzzy":17},"total_outbound_references":41},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"thesis":"As of 16 August 2026, this Paper Citation Record lists 41 of 41 outbound references and 2 inbound Pith citation observations for arXiv:2501.00257."}