{"as_of":"2026-08-11T07:49:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:ff1ef9c9e819da024efd5fd70868e0f6ca0dd89aac58d89985e384cdbce8335f","coverage":[{"denominator":94,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":94,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-06T13:38:01.264257Z","state":"measured"},{"denominator":94,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":94,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-11T06:34:44.6726+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2507.20419/citation-record","integrity":"/paper/2507.20419/integrity","json":"/paper/2507.20419/citation-record.json","paper":"/paper/2507.20419"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:51.994781Z","title":"Principles of Evaluation in Natural Language Processing,","venue":null,"work_id":null,"year":2007},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:51.994781Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:ed02fca6d634b8e1441e8940e82f1752f9af04c765c5a99e65a4e9a5645fbbbc","observation_id":"689c45d2-87cc-4baa-a256-e26be0f53ed6","resolution":{"observed_at":"2026-08-06T13:37:51.994781Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:52.365652Z","title":"Natural Language Inference ,","venue":null,"work_id":null,"year":1973},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:52.365652Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:1701bc2e532906addfbd49a47fc22946ad1b665e2cb791c324bf218fc87caf75","observation_id":"7680c4c5-ab05-40de-bfb9-5b3bdd50f0df","resolution":{"observed_at":"2026-08-06T13:37:52.365652Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:52.487079Z","title":"PROBABILISTIC TEXTUAL ENTAILMENT: GENERIC APPLIED MODELING OF LANGUAGE VARIABILITY,","venue":null,"work_id":null,"year":2004},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:52.487079Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:cfce16ad22892f1d25b31009f18945474faeb2c2a4139abd5b75f2c0db88918d","observation_id":"1259cf48-029a-46e1-a7d5-840d469b0b86","resolution":{"observed_at":"2026-08-06T13:37:52.487079Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:52.725915Z","title":"Probabilistic textual entailment: Generic applied modeling of language variability,","venue":null,"work_id":null,"year":2004},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:52.725915Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:9cd09ba8d80490cd65976f0df9cdf1e8825b18461ff1c7a2e6ac9b1882111587","observation_id":"927f39e9-f3cc-454e-bf7d-d4a33ded723b","resolution":{"observed_at":"2026-08-06T13:37:52.725915Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:52.872893Z","title":"The Seventh PASCAL Recognizing Textual Entailment Challenge,","venue":null,"work_id":null,"year":2011},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:52.872893Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:7ee9e57f6ee333cc2e1d1cdce375d8328d8d13d40f8d6e8c73aacc12e1216cc7","observation_id":"f96c17a0-920b-4d4d-8f46-579190916b18","resolution":{"observed_at":"2026-08-06T13:37:52.872893Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:53.153624Z","title":"The Sixth PASCAL Recognizing Textual Entailment Challenge,","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:53.153624Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:b5e4ae78b582643481210333136ae7865976b2c3ec95a991c9b440adc44cb567","observation_id":"55544bb1-66cf-4be4-b593-6d34d9c0c05d","resolution":{"observed_at":"2026-08-06T13:37:53.153624Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:53.279800Z","title":"The Fifth PASCAL Recognizing Textual Entailment Challenge,","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:53.279800Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:3d56f2747c7fcb1841e615e4f2dec988406e44695e652e0d708a925c077cf581","observation_id":"7fe02df0-38d3-44dc-8c54-1e59a0b1d7b2","resolution":{"observed_at":"2026-08-06T13:37:53.279800Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:53.425491Z","title":"The Third PASCAL Recognizing Textual Entailment Challenge,","venue":null,"work_id":null,"year":2007},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:53.425491Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:c30e54f7197c803e3aeaabccb3c5cc406c08531a1cab3c45d20543a215337866","observation_id":"9ece044d-c086-4df3-bf6b-0e44cb3cd691","resolution":{"observed_at":"2026-08-06T13:37:53.425491Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:53.601192Z","title":"The Second PASCAL Recognising Textual Entailment Challenge,","venue":null,"work_id":null,"year":2006},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:53.601192Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:89866d4b35b1c59bfaddd0d67340a1173d3a863ca232a393746713380f5b8133","observation_id":"92ba844e-0949-47ea-addb-9ab860166dde","resolution":{"observed_at":"2026-08-06T13:37:53.601192Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:53.769882Z","title":"The PASCAL Recognising Textual Entailment Challenge,","venue":null,"work_id":null,"year":2006},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:53.769882Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:c41eb7f58fad7ed1dd187f2cacb61108903f6ba314f1ef0b9c04b08abf064e48","observation_id":"1b9991c9-4ae9-4ed4-88b2-123b54dcb7e0","resolution":{"observed_at":"2026-08-06T13:37:53.769882Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:53.910758Z","title":"The Fourth PASCAL Recognizing Textual Entailment Challenge,","venue":null,"work_id":null,"year":2008},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:53.910758Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:7879f511f123bc3403e0ce8d091380fced29bc2072f43cd05f8517411a528a7f","observation_id":"cae6e79b-fc41-44fd-9fbe-c77460cc6ebf","resolution":{"observed_at":"2026-08-06T13:37:53.910758Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.2200/s00509ed1v01y201305hlt023","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"Recognizing Textual Entailment: Models and Applications,","venue":"Synthesis lectures on human language technologies","work_id":"741721d2-27d1-4c6d-98ce-e16bf33b7631","year":2013},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.002707Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:cc490a047f360e2cf5f94fe804a84581017d60d23e56ea4af472baae57ace1d5","observation_id":"f78a0d00-f5f3-4b3f-b020-b63fb381a23b","resolution":{"observed_at":"2026-08-06T13:38:04.882356Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:54.120712Z","title":"The Winograd Schema Challenge,","venue":null,"work_id":null,"year":2011},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.120712Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:3ad607919bdf7efd07bcef088c57b6e0e790a2f8b7e3b1cca5b1f70e616a53f5","observation_id":"ebe93d99-08dc-4bbf-b71a-f2a226fd7535","resolution":{"observed_at":"2026-08-06T13:37:54.120712Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:54.187851Z","title":"A Broad-Coverage Challenge Corpus for Sentence Understanding through Inference,","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.187851Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:870a8536bba03fac5aeb1c923500bea9b73567e129c61ed1d92f86d4470db45b","observation_id":"08857552-76f4-423c-8d58-10b41f0d7ba5","resolution":{"observed_at":"2026-08-06T13:37:54.187851Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:54.254334Z","title":"A large annotated corpus for learning natural language inference,","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.254334Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:009b8af9820147c2a813b696899d91552f6db4ec052e9ca7b49928931d496f47","observation_id":"66995a61-e64d-4e3c-9ff5-05e584fbf5a5","resolution":{"observed_at":"2026-08-06T13:37:54.254334Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:54.336819Z","title":"GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding,","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.336819Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:2948907b397618848042607a6c778a656672799c19d4afc0393af92a6f44c0bb","observation_id":"3597f193-4c39-413e-ad2a-facdc16169b5","resolution":{"observed_at":"2026-08-06T13:37:54.336819Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2204.02311","last_updated":"2022-10-05T06:02:24Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-05T16:11:45Z","title":"PaLM: Scaling Language Modeling with Pathways","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.02311","snapshot_observed_at":"2026-08-06T13:37:54.375277Z","title":"PaLM: Scaling Language Modeling with Pathways,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.375277Z"},"links":{"cited_paper":"/paper/2204.02311","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:4367d7689e9327a093b0dcf21f18ddaf80d147a63ed5a85590a9003eefb2f726","observation_id":"f1d0f05a-eadd-4e8d-b8ea-4d6e4d976f47","resolution":{"observed_at":"2026-08-06T13:37:54.375277Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2212.01853","last_updated":"2022-12-04T15:36:18Z","snapshot_observed_at":"2026-08-10T12:39:34.606308Z","submitted_at":"2022-12-04T15:36:18Z","title":"Toward Efficient Language Model Pretraining and Downstream Adaptation via Self-Evolution: A Case Study on SuperGLUE","version":1},"cited_work":{"arxiv_id":"2212.01853","doi":null,"metadata_source":"pith","pith_arxiv_id":"2212.01853","snapshot_observed_at":"2026-08-06T13:38:06.324880Z","title":"Toward Efficient Language Model Pretraining and Downstream Adaptation via Self-Evolution: A Case Study on SuperGLUE","venue":"cs.CL","work_id":"15650321-4431-4899-8f85-54caa9bfb14a","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.473408Z"},"links":{"cited_paper":"/paper/2212.01853","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:d6b546bcc7a559112b17d04659d6e90efd7d61caa8cc919dba155288df001a1e","observation_id":"77c1d34e-f5b0-49df-a804-4d570799d029","resolution":{"observed_at":"2026-08-06T13:38:06.437083Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.812775Z","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach,","venue":null,"work_id":"7665aec6-b162-44d1-ad55-165aacd9d81c","year":2019},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.552526Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:fc41f37eacf5c8d9057fc05240b76efe31ff4135c32a13bbd7b52a192ff259e6","observation_id":"3368c16d-2b2d-4928-be27-0ec4dcc1c3e8","resolution":{"observed_at":"2026-08-06T13:38:07.816504Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.799883Z","title":"Semantics-aware BERT for Language Understanding,","venue":null,"work_id":"33c2c9d3-ddb9-4404-b6b1-9d4dc3904614","year":2019},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.618700Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:1b68f13b1bae99b41dcbfcb3ba6a3dfe1fdf56b18ff29f34217bc3746ddf48fa","observation_id":"bb834d31-58b1-488c-a015-781ebdc0d131","resolution":{"observed_at":"2026-08-06T13:38:07.803775Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.787239Z","title":"XLNet: Generalized Autoregressive Pretraining for Language Understanding,","venue":null,"work_id":"4c70d31f-f2e5-44d5-b88d-6d7d1fb808c5","year":2019},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.677424Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:c266e2b668289863e40bbfe210622260341e3dc4735b43afe64f137f98ae8388","observation_id":"6d1c5b9b-b0dd-4cf2-8f26-47ae3d0b1789","resolution":{"observed_at":"2026-08-06T13:38:07.791260Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2208.01448","last_updated":"2022-08-03T12:42:52Z","snapshot_observed_at":"2026-08-11T01:35:03.956807Z","submitted_at":"2022-08-02T13:30:07Z","title":"AlexaTM 20B: Few-Shot Learning Using a Large-Scale Multilingual Seq2Seq Model","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2208.01448","snapshot_observed_at":"2026-08-06T13:37:54.767217Z","title":"AlexaTM 20B: Few-Shot Learning Using a Large-Scale Multilingual Seq2Seq Model,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.767217Z"},"links":{"cited_paper":"/paper/2208.01448","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:e744967f9186022b10e865cd13e796a19e15b0cdb47d1ee0432c7c61e25b8994","observation_id":"92347395-5567-476b-b319-f3dd66da57aa","resolution":{"observed_at":"2026-08-06T13:37:54.767217Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.17564","last_updated":"2023-12-21T06:21:11Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-03-30T17:30:36Z","title":"BloombergGPT: A Large Language Model for Finance","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.17564","snapshot_observed_at":"2026-08-06T13:37:54.862401Z","title":"BloombergGPT: A Large Language Model for Finance,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.862401Z"},"links":{"cited_paper":"/paper/2303.17564","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:848f7e68b9df3b8b88da98258e979769b4316de05fa01ca47ea2abc1d9aa9274","observation_id":"c865e660-91f3-44ca-8312-1150c6491cd4","resolution":{"observed_at":"2026-08-06T13:37:54.862401Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.09263","last_updated":"2024-12-13T06:28:11Z","snapshot_observed_at":"2026-08-09T23:46:14.763674Z","submitted_at":"2024-12-12T13:21:09Z","title":"First Train to Generate, then Generate to Train: UnitedSynT5 for Few-Shot NLI","version":2},"cited_work":{"arxiv_id":"2412.09263","doi":"10.48550/arxiv.2412.09263","metadata_source":"pith","pith_arxiv_id":"2412.09263","snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"First Train to Generate, then Generate to Train: UnitedSynT5 for Few-Shot NLI","venue":"cs.CL","work_id":"8101b417-2b82-44eb-a441-b4feef34ab86","year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.934359Z"},"links":{"cited_paper":"/paper/2412.09263","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:5a423655da8813e6b99bc965849109fab42c839246887447ff85c142b5ff208f","observation_id":"3ff88a52-1719-405a-80ac-5075153207c8","resolution":{"observed_at":"2026-08-06T13:38:04.472952Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:54.981376Z","title":"ByT5: Towards a Token-Free Future with Pre-trained Byte-to-Byte Models,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:54.981376Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:66f25fdc793e39709468fcfea11f643edf46c57d966b8b5bac0cd844e7955958","observation_id":"a6ec2638-a7ae-49e6-968c-5deaf8a1b8fe","resolution":{"observed_at":"2026-08-06T13:37:54.981376Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2010.12821","last_updated":"2020-10-24T07:43:00Z","snapshot_observed_at":"2026-07-06T10:08:08.004416Z","submitted_at":"2020-10-24T07:43:00Z","title":"Rethinking embedding coupling in pre-trained language models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2010.12821","snapshot_observed_at":"2026-08-06T13:37:55.076184Z","title":"Rethinking embedding coupling in pre-trained language models,","venue":null,"work_id":null,"year":2010},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.076184Z"},"links":{"cited_paper":"/paper/2010.12821","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:f16020ff73094d0ae72e431ba8b584792618245bfba2a516e432bb0b799f48c7","observation_id":"3fa32c45-f2d4-4d46-9df6-b83f036f226a","resolution":{"observed_at":"2026-08-06T13:37:55.076184Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:55.158177Z","title":"mGPT: Few-Shot Learners Go Multilingual,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.158177Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:91ce0661fa728a2a69e73ba239889de523cf1c5015489337afec18cef37a6be3","observation_id":"5b3cc215-56f3-4f27-9ab0-4497e504c3bd","resolution":{"observed_at":"2026-08-06T13:37:55.158177Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.774270Z","title":"DeBERTa: Decoding-enhanced BERT with Disentangled Attention,","venue":null,"work_id":"01aa2202-8cf2-4dc1-abb8-f83fba8b3ecf","year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.202142Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:eb308a9217fc861f505147c18bae056d3cf1f2d23892a48b594997e95a0a3df3","observation_id":"c363be4e-caae-41c6-acfb-d19496db7f62","resolution":{"observed_at":"2026-08-06T13:38:07.778137Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:52.167222Z","title":"Available: https://aclanthology.org/2007.tal-1.1","venue":null,"work_id":null,"year":2007},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:52.167222Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:84d1aedc683cdc587c5784e830a3efb821d923c150339d64176fc980542227ed","observation_id":"c9666e37-80af-4a35-9a5a-f5a4ba0d4a8e","resolution":{"observed_at":"2026-08-06T13:37:52.167222Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:55.271734Z","title":"SpanBERT: Improving Pre-training by Representing and Predicting Spans,","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.271734Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:102155da2b349f7c3c2817a5c03ae039413bc3a90b7c5b474cc77a0fba1e1d16","observation_id":"5f610143-a1b9-4151-adf0-e3e4041827f1","resolution":{"observed_at":"2026-08-06T13:37:55.271734Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2020.sustainlp-1.17","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"SqueezeBERT: What can computer vision teach NLP about efficient neural networks?,","venue":null,"work_id":"6e3c8a62-be6d-422d-9dcd-d2f8a7aaa15a","year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.343223Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:fcbb0f5936b6fe1ae3e6e961f971c4d40e8e5ed51b6bf17ea83afbefd1b4b107","observation_id":"282848ce-90e3-4530-9594-25e0b0ed6caa","resolution":{"observed_at":"2026-08-06T13:38:04.069737Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1910.01108","last_updated":"2020-03-01T02:57:50Z","snapshot_observed_at":"2026-08-07T19:07:36.327251Z","submitted_at":"2019-10-02T17:56:28Z","title":"DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1910.01108","snapshot_observed_at":"2026-08-06T13:37:55.387682Z","title":"DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter,","venue":null,"work_id":null,"year":1910},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.387682Z"},"links":{"cited_paper":"/paper/1910.01108","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:f0f9b08e976faf446557b2db9136a29f914cb6ba7f61e60933955e9dd41c37af","observation_id":"9a9672f9-0a89-4189-b7b4-577db920ad26","resolution":{"observed_at":"2026-08-06T13:37:55.387682Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.761477Z","title":"A Dataset for Arabic Textual Entailment,","venue":null,"work_id":"0a795e51-c17d-4ddb-8eb2-9c0d0abb9036","year":2013},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.479023Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:59c0c601f6f5dd7500fe3b46cf9dcbe62a09c3e02ce2282d3071b3c75a5c9d98","observation_id":"ee6d3f64-5603-453d-9f90-1a7a75c00cf4","resolution":{"observed_at":"2026-08-06T13:38:07.765397Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.7494/csci.2023.24.2.4378","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"ARNLI: ARABIC NATURAL LANGUAGE INFERENCE ENTAILMENT AND CONTRADICTION DETECTION,","venue":"Computer Science","work_id":"2c7c0201-c157-4ec0-b506-613dbd14d488","year":2023},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.613730Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:a6f6287a73f78c7f6bbf7b039c0004be3d2fb652614b1096ab8aa22924e5f204","observation_id":"3278f0e6-242a-41c4-b4e6-b98a3facfe33","resolution":{"observed_at":"2026-08-06T13:38:03.913885Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1007/s10579-024-09731-1","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:03:40.895421Z","title":"ArEntail: manually-curated Arabic natural language inference dataset from news headlines,","venue":"Language Resources and Evaluation","work_id":"de78ad1a-5732-4cd3-afcd-5ea9d35ad8bd","year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.710306Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:7c58139af29928d002df2daca7c9fa2f9123cb0402d99bd7ca1d43673abb675b","observation_id":"bd9dd812-c1de-45af-9eb6-6662e3eb03f9","resolution":{"observed_at":"2026-08-06T13:38:03.723052Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.749025Z","title":"Baselines and Test Data for Cross-Lingual Inference,","venue":null,"work_id":"a3332068-8b96-4621-8d40-12330e681070","year":2018},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.827796Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:fd4a5b7373264c8dd70621a662298fd8fa6d003e1defb97840e79a43efe03593","observation_id":"1a673ff0-d2b4-49a6-98f6-a8169e2a117e","resolution":{"observed_at":"2026-08-06T13:38:07.752969Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:55.900030Z","title":"XNLI: Evaluating Cross-lingual Sentence Representations,","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.900030Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:94b45f69c00a2b80dbdcedf23da24b263c4a2cfe419f758a48203d84e598d547","observation_id":"72509850-399e-4755-a1a4-301479c834bc","resolution":{"observed_at":"2026-08-06T13:37:55.900030Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:55.976655Z","title":"Benchmarking Zero-shot Text Classification: Datasets, Evaluation and Entailment Approach,","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:55.976655Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:d717f4bd25ff02cb1981f1bab59483a36d279d1f0b924e6e4c5ef50a0927fed5","observation_id":"5eef9a04-89dc-4f4f-915a-623b072993da","resolution":{"observed_at":"2026-08-06T13:37:55.976655Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:56.072296Z","title":"BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension,","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.072296Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:bfdda9b1c1c899845e00ad92de9927f323f33afd053c1128822ea7f37cd7aab7","observation_id":"fed28c6d-80b1-4c58-a42d-1a3e195df645","resolution":{"observed_at":"2026-08-06T13:37:56.072296Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1907.11692","last_updated":"2019-07-26T17:48:29Z","snapshot_observed_at":"2026-07-31T22:31:37.910868Z","submitted_at":"2019-07-26T17:48:29Z","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1907.11692","snapshot_observed_at":"2026-08-06T13:37:56.192293Z","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach,","venue":null,"work_id":null,"year":1907},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.192293Z"},"links":{"cited_paper":"/paper/1907.11692","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:de9503c6d351c473a30ae6ff6025e68ab4b5cea78a590698b5b90e15b4e686ef","observation_id":"134ae3ce-000c-4867-b024-63f1140fcd0f","resolution":{"observed_at":"2026-08-06T13:37:56.192293Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2002.10957","last_updated":"2020-04-06T02:53:18Z","snapshot_observed_at":"2026-08-10T12:28:10.970434Z","submitted_at":"2020-02-25T15:21:10Z","title":"MiniLM: Deep Self-Attention Distillation for Task-Agnostic Compression of Pre-Trained Transformers","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2002.10957","snapshot_observed_at":"2026-08-06T13:37:56.242266Z","title":"MiniLM: Deep Self-Attention Distillation for Task- Agnostic Compression of Pre-Trained Transformers,","venue":null,"work_id":null,"year":2002},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.242266Z"},"links":{"cited_paper":"/paper/2002.10957","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:5ff2fcf09f48e8cb102794af6447dc60765c57335de35ead1edf11bf3b48b6de","observation_id":"4a4fdf83-36d2-47a1-9a3a-bf5c34d8e745","resolution":{"observed_at":"2026-08-06T13:37:56.242266Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2111.09543","last_updated":"2023-03-24T09:17:17Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2021-11-18T06:48:00Z","title":"DeBERTaV3: Improving DeBERTa using ELECTRA-Style Pre-Training with Gradient-Disentangled Embedding Sharing","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2111.09543","snapshot_observed_at":"2026-08-06T13:37:56.318397Z","title":"DeBERTaV3: Improving DeBERTa using ELECTRA-Style Pre-Training with Gradient- Disentangled Embedding Sharing,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.318397Z"},"links":{"cited_paper":"/paper/2111.09543","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:6474bb640cb8ff1dac322b7abf6a874154600ecceead5a9d3fb06e10dec82829","observation_id":"6e8521c3-0535-431a-adb0-cf7b87e61d38","resolution":{"observed_at":"2026-08-06T13:37:56.318397Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:56.411095Z","title":"Less annotating, more classifying: Addressing the data scarcity issue of supervised machine learning with deep transfer learning and BERT-NLI,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.411095Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:d123dccc467a5fa0846db0a9386e55196eb24e751818f2e935b10fa81a442c98","observation_id":"16c9bc1a-6cad-42d4-b394-b7fbd6ae360d","resolution":{"observed_at":"2026-08-06T13:37:56.411095Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:56.506650Z","title":"GPTAraEval: A Comprehensive Evaluation of ChatGPT on Arabic NLP,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.506650Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:660509687480609450d749dc3c38a1e819b6d5fd8b64cc11b359a6d0e32dd5ad","observation_id":"667d3618-e2f9-4aad-aed6-1437fbae2362","resolution":{"observed_at":"2026-08-06T13:37:56.506650Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2022.gem-1.6","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"CLSE: Corpus of Linguistically Significant Entities,","venue":null,"work_id":"69dc3717-405e-4737-aea9-aec552e180cd","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.576929Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:0431b9633ed676759ae1ffdb55964b0ae65fc8028f9741619bc049c906f0c545","observation_id":"40349f99-cd76-426c-8308-65af190c6935","resolution":{"observed_at":"2026-08-06T13:38:03.524186Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:56.721456Z","title":"The GEM Benchmark: Natural Language Generation, its Evaluation and Metrics,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.721456Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:304ab66f65b728f001dd5da14eca3870db2be4ad411567d299644aa1fd68e708","observation_id":"6780fe5d-533e-4a91-b726-fad1dc3b9773","resolution":{"observed_at":"2026-08-06T13:37:56.721456Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:56.833242Z","title":"GEMv2: Multilingual NLG Benchmarking in a Single Line of Code,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.833242Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:10518bda17158f6a5d1f7d356c6444214924912918c70d92a1de472aac339d33","observation_id":"d9bdb60a-d4bf-4124-a838-90cf873f7543","resolution":{"observed_at":"2026-08-06T13:37:56.833242Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2022.emnlp-main.360","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"IndicNLG Benchmark: Multilingual Datasets for Diverse NLG Tasks in Indic Languages,","venue":null,"work_id":"668088ee-1426-4e06-9747-44a8d37ba8b9","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:56.920313Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:2d80aa2277c1b686ba0710c701dd28c74e6c37ccba84065aefd2b6ecb4f83b6f","observation_id":"8d70fd07-bf64-4b58-9148-ee1014d64153","resolution":{"observed_at":"2026-08-06T13:38:03.343289Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:57.001448Z","title":"MTG: A Benchmark Suite for Multilingual Text Generation,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.001448Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:d02b66bddb2876d5cb50be5bd6ad70126be0e42b8192cabfbe66c7b6e9cc244b","observation_id":"cefb55ee-4493-4e6d-a886-cfd1123f63fa","resolution":{"observed_at":"2026-08-06T13:37:57.001448Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:57.121190Z","title":"IndoNLG: Benchmark and Resources for Evaluating Indonesian Natural Language Generation,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.121190Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:5c05ab1a26a4519fc3553a2565e7984acc7f55b367dbb3be44ba08f7937827e9","observation_id":"d9de5286-75d9-4675-8b07-727b8602c4a2","resolution":{"observed_at":"2026-08-06T13:37:57.121190Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.735726Z","title":"Dolphin: A Challenging and Diverse Benchmark for Arabic NLG,","venue":null,"work_id":"6bc2c930-f215-4b49-9331-60957cd943e7","year":2023},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.234294Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:18c8e6c5f2732b915aafbc35cd984b8020b04f996994516a58e4e70c1b41eda8","observation_id":"5bcbe44a-2669-4ef9-a89d-6eb813094b39","resolution":{"observed_at":"2026-08-06T13:38:07.740001Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.722853Z","title":"TURJUMAN: A Public Toolkit for Neural Arabic Machine Translation,","venue":null,"work_id":"067c0c5e-25d1-44f3-b3c4-ce4b1d54aeb9","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.449397Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:934594317f6c670ade2ec156b97a4b3a5f67c74df504ff745e5c6a593e6fb832","observation_id":"8720e990-6952-456b-85f3-0f4c21cbe7cb","resolution":{"observed_at":"2026-08-06T13:38:07.727123Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:57.376864Z","title":"AraBench: Benchmarking Dialectal Arabic-English Machine Translation,","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.376864Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:604fdd667805afd3ebb84c60147c9dfe1dfd58f62ad31fd6330d884d7c6f4038","observation_id":"ddab1a97-4f81-4959-a22c-a35ea5dffcae","resolution":{"observed_at":"2026-08-06T13:37:57.376864Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:57.717646Z","title":"BanglaNLG and BanglaT5: Benchmarks and Resources for Evaluating Low-Resource Natural Language Generation in Bangla,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.717646Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:4e30e1d53ab60f405119d60e2a9fe1374ff94dcc01a8c5a67b675729bcf48928","observation_id":"71743e22-ff20-481c-9453-e97fd137ca06","resolution":{"observed_at":"2026-08-06T13:37:57.717646Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2022.acl-long.47","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"AraT5: Text-to-Text Transformers for Arabic Language Generation,","venue":"Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","work_id":"3f98c350-2bc3-4cce-a797-5c218556f7ac","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.586733Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:43ad5450a3b2c863a99302cab2b8e7a3f7d2876c08d65a2340914050419cc3eb","observation_id":"e9f11951-f937-49e1-9aee-a16f37f9f3d7","resolution":{"observed_at":"2026-08-06T13:38:03.156193Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2112.13610","last_updated":"2022-06-14T07:19:35Z","snapshot_observed_at":"2026-08-06T04:10:30.783266Z","submitted_at":"2021-12-27T11:08:58Z","title":"CUGE: A Chinese Language Understanding and Generation Evaluation Benchmark","version":2},"cited_work":{"arxiv_id":"2112.13610","doi":null,"metadata_source":"pith","pith_arxiv_id":"2112.13610","snapshot_observed_at":"2026-08-06T13:38:05.760935Z","title":"CUGE: A Chinese Language Understanding and Generation Evaluation Benchmark","venue":"cs.CL","work_id":"165e321a-4b4d-4ba6-81fe-dfa4b92d8cb8","year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.924811Z"},"links":{"cited_paper":"/paper/2112.13610","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:d640abbc1053bccb9c02d43c5bd6f8ddca846541635f4da9107cea83d3d6f9eb","observation_id":"dfb6a474-923b-401a-b713-bd14cc433a19","resolution":{"observed_at":"2026-08-06T13:38:05.964795Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2021.emnlp-main.369","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"PhoMT: A High-Quality and Large-Scale Benchmark Dataset for Vietnamese-English Machine Translation,","venue":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","work_id":"dffde7be-5c12-4987-945f-0166b91a8fc8","year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.828392Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:6cb413ae213172380983afcc7829bb664c634dedbb834883618e394e45b1ee35","observation_id":"697a2908-5568-4ec5-aa20-637db9fab701","resolution":{"observed_at":"2026-08-06T13:38:02.957624Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.710413Z","title":"Benchmarking Multidomain English-Indonesian Machine Translation,","venue":null,"work_id":"bdf0d404-1a27-4082-a32b-5a16bc0ea216","year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.142558Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:0c690c886cb3a2e0e00ec90ab60c230d0938c860165fa5ee7b7cdea41d4b758b","observation_id":"6026dce6-d303-4a06-95d5-120a540ddcfb","resolution":{"observed_at":"2026-08-06T13:38:07.714116Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:57.998337Z","title":"LOT: A Story-Centric Benchmark for Evaluating Chinese Long Text Understanding and Generation,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.998337Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:5b2943366feb75038da5a3f9f1f97b39574c303c0f2832df224f3ba23a30d59f","observation_id":"733fc5c9-80fc-4805-99d5-8ee6d3a4d02c","resolution":{"observed_at":"2026-08-06T13:37:57.998337Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.697742Z","title":"XGLUE: A New Benchmark Dataset for Cross-lingual Pre-training, Understanding and Generation,","venue":null,"work_id":"c25d6efb-5c22-459b-9ba4-8ef7c1470418","year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.317146Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:fefb8ad33995752fcbcd8edbb0b91d56f328f3dd7a87b82f7c77b5b4a008fe46","observation_id":"ec315073-34a0-4046-908d-7d5c555da901","resolution":{"observed_at":"2026-08-06T13:38:07.701733Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2021.findings-acl.36","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"GLGE: A New General Language Generation Evaluation Benchmark,","venue":null,"work_id":"e1696b0e-c849-45c5-87bc-9f1646f44f2d","year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.240465Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:fcafd277de974048aba208cf7e65ca0c071a86bf47c36aee4056373f5901a6c1","observation_id":"834cf8cc-1dd7-40e3-a06a-18e24224514f","resolution":{"observed_at":"2026-08-06T13:38:02.737908Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:58.638320Z","title":"XTREME-R: Towards More Challenging and Nuanced Multilingual Evaluation,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":64,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.638320Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:dcdf1142957e34b0edff72a8bab434cdb87fb0bf33d5067f4f70bc30daa6590f","observation_id":"cd68d2a5-e9de-4fd4-84b3-337170cf595c","resolution":{"observed_at":"2026-08-06T13:37:58.638320Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.672356Z","title":"SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems,","venue":null,"work_id":"64a60517-0290-4df0-af69-bf187105c3ac","year":2019},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":65,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.735056Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:7445c0a22b99cf6c76a1a528d5224c8875dd5a4eb5b0fef5e7e40c7bfe861f21","observation_id":"5d447d3c-7d5e-4330-9a75-7b9517c6d21f","resolution":{"observed_at":"2026-08-06T13:38:07.676377Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.685073Z","title":"XTREME: A Massively Multilingual Multi-task Benchmark for Evaluating Cross-lingual Generalization,","venue":null,"work_id":"78d7922a-f724-4e1f-91a0-2ad93745d08b","year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":66,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.530760Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:87e6ef98dc76407439d9846b8fd90134364bc2957a9b099b5672aa588193c48e","observation_id":"b22bb799-41d6-4867-8b87-9c6c7755d77d","resolution":{"observed_at":"2026-08-06T13:38:07.689029Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:58.891569Z","title":"ARBERT & MARBERT: Deep Bidirectional Transformers for Arabic,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":67,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.891569Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:c3b2a7850faf43ab2ba4913041eeb1bbbdd283f76c804f50cca10e6af01279a6","observation_id":"91ba2886-2bae-4cb1-b74f-6be8ae3ec06f","resolution":{"observed_at":"2026-08-06T13:37:58.891569Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.647310Z","title":"ORCA: A Challenging Benchmark for Arabic Language Understanding,","venue":null,"work_id":"4101760b-245f-49be-b862-a65faa2428d1","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":68,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.994414Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:ce52ccae859b478702963913c080c9f8a7088c784b99bd3f0331287636e8c135","observation_id":"6c24f0c1-3e5d-44a9-bc2a-ea910c5188b6","resolution":{"observed_at":"2026-08-06T13:38:07.651628Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.659997Z","title":"ALUE: Arabic Language Understanding Evaluation,","venue":null,"work_id":"5cf6dce6-c069-421f-8b0e-9590f0bc556e","year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":69,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.815279Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:df2ca547e34954bb54785404266f19f18db907c17bf1f089cf538169c9e64f40","observation_id":"b32b011a-7ce8-4541-b6bd-ae0eff8c7655","resolution":{"observed_at":"2026-08-06T13:38:07.663782Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.634024Z","title":"LAraBench: Benchmarking Arabic AI with Large Language Models,","venue":null,"work_id":"860f2470-5e0c-4177-89cc-a7fcf2ae698a","year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":70,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.258663Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:459233ea391aab4699a6bac58407bd949d6b7b092f3cc709f474a9769060694d","observation_id":"aa3195d2-a51d-4dc0-b2af-9a6328055803","resolution":{"observed_at":"2026-08-06T13:38:07.638619Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1502.05698","last_updated":"2015-12-31T13:08:14Z","snapshot_observed_at":"2026-07-06T04:09:45.600447Z","submitted_at":"2015-02-19T20:46:10Z","title":"Towards AI-Complete Question Answering: A Set of Prerequisite Toy Tasks","version":10},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1502.05698","snapshot_observed_at":"2026-08-06T13:37:59.448235Z","title":"Towards AI-Complete Question Answering: A Set of Prerequisite Toy Tasks,","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":71,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.448235Z"},"links":{"cited_paper":"/paper/1502.05698","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:733f171e87cda34429d49a7f769ff81afd7f3946979c9f0f9aafe454688478cd","observation_id":"d53b659a-8b8c-4b69-b0d3-297b7cee87ca","resolution":{"observed_at":"2026-08-06T13:37:59.448235Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:59.160290Z","title":"ArabicMMLU: Assessing Massive Multitask Language Understanding in Arabic,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":72,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.160290Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:d5153b73b877f7bd9f95659824081c838ad0cca2454cbc8da8157cf4fa24fe36","observation_id":"a748ce81-d50b-4170-9c70-a9b4a5157d34","resolution":{"observed_at":"2026-08-06T13:37:59.160290Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2020.findings-emnlp.39","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"KorNLI and KorSTS: New Benchmark Datasets for Korean Natural Language Understanding,","venue":null,"work_id":"32543dea-2333-4d26-b14d-c05bef0db2f8","year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":73,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.675564Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:d7dcd8134d5f42a6b33e4ee48567a34b2a8b544143cfe4d47aaf5f85e2874fb2","observation_id":"9877f715-1110-4b79-8c3f-227868f75d03","resolution":{"observed_at":"2026-08-06T13:38:02.509856Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:59.782498Z","title":"CLUE: A Chinese Language Understanding Evaluation Benchmark,","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":74,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.782498Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:6dee6ebbb465d30a6cc50c8fd9da30e1aafbbe7122378921b36bd826c69e9b73","observation_id":"9a5d9f2e-eccc-4638-a16a-58b6e0d13866","resolution":{"observed_at":"2026-08-06T13:37:59.782498Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.592642Z","title":"JGLUE: Japanese General Language Understanding Evaluation,","venue":null,"work_id":"f8808623-3a4d-4b01-b296-a60290706471","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":75,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.854991Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:499f5fb514b529dbab650a748a0fd277128b94ebaca8f9f132bf8132c8250687","observation_id":"3c0eb517-fdff-4864-8015-0a2649c2db02","resolution":{"observed_at":"2026-08-06T13:38:07.597294Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.606742Z","title":"FlauBERT: Unsupervised Language Model Pre-training for French,","venue":null,"work_id":"be269009-c465-4feb-b746-b7abe9e2f282","year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":76,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.569273Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:ba789340e7cb27481cf4984a42b027ea32493bad3ae1bab50d47c02dc3537ee8","observation_id":"1a814e55-d8bb-433c-aca1-1e236472a633","resolution":{"observed_at":"2026-08-06T13:38:07.611359Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.579262Z","title":"KLUE: Korean Language Understanding Evaluation,","venue":null,"work_id":"2aa97214-e283-4dae-b14f-f028a121e1a7","year":2021},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":77,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.038452Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:b204d0f02389d8d52d1478017eddfb8610c28f75284abb461ef7d043d52d7af2","observation_id":"41de1071-0512-47c3-a8fe-401ca88f2706","resolution":{"observed_at":"2026-08-06T13:38:07.583677Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2023.acl-demo.33","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"UINAUIL: A Unified Benchmark for Italian Natural Language Understanding,","venue":null,"work_id":"2266952a-ab46-4eb4-b8d8-d200c03736b5","year":2023},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":78,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.166860Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:65501390be0edac321817cb57cd177574439ff82325fb41d79e56e1e721ff59e","observation_id":"1f389aa9-9204-45a9-b2f4-5567479ee156","resolution":{"observed_at":"2026-08-06T13:38:02.321727Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:00.244011Z","title":"SuperGLEBer: German Language Understanding Evaluation Benchmark,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":79,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.244011Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:0523280e29153e1c7d94c708c65b41f151359e2f22df977e8ec53bb3fafa3125","observation_id":"f7a1f880-dcd6-4806-b4b3-97735c014b95","resolution":{"observed_at":"2026-08-06T13:38:00.244011Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:59.916339Z","title":"IndoNLU: Benchmark and Resources for Evaluating Indonesian Natural Language Understanding,","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":80,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.916339Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:b80325b8f62cc27dc6228eff1548242e4d3b3cecf8c98f7c22f7920a6ae2261e","observation_id":"8d1de291-c078-4bfd-a723-c1c8c9e3acb6","resolution":{"observed_at":"2026-08-06T13:37:59.916339Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.565594Z","title":"IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages,","venue":null,"work_id":"4fd9abd1-70cd-4e75-8042-77b5293c3959","year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":81,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.370586Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:2707b02151ec769315ec93e571a7533e103d7a435b466059248505f8db5e2522","observation_id":"76900af7-102a-4306-8a0b-67ccdec9f9c5","resolution":{"observed_at":"2026-08-06T13:38:07.569610Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.551707Z","title":"Using the Framework,","venue":null,"work_id":"eb2505a6-270d-4917-a1c6-4c9e045ed693","year":1996},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":82,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.526646Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:6351c3d94d445fc026dfca571df26b8e559dbec6d77ccd8573dcdc6e9ef014e0","observation_id":"1e878b29-8a63-4574-a305-5822fc162d67","resolution":{"observed_at":"2026-08-06T13:38:07.555603Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.537345Z","title":"An extended model of natural logic","venue":null,"work_id":"8d21125b-403a-4c7f-ae67-e16926ea2c78","year":null},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":83,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.643216Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:82baab7938fea466e78737aecf291a2bbf513dc9bc1c4a80fa6419e5fe5ce55f","observation_id":"ba872268-3e9f-40d9-9560-5bbaf7689c15","resolution":{"observed_at":"2026-08-06T13:38:07.542099Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2024.findings-naacl.15","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"VLUE: A New Benchmark and Multi-task Knowledge Transfer Learning for Vietnamese Natural Language Understanding,","venue":null,"work_id":"8e2a3576-4b63-4c3b-a861-582dc9ba6808","year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":84,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.314390Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:7811fb27e2e8895f42ffde46fa4938dcd60e60aa261d230ad2a2deb1fc3e1c35","observation_id":"200d0496-5d0a-4469-b32f-05bc9bee392d","resolution":{"observed_at":"2026-08-06T13:38:02.179538Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:05.304873Z","title":"Analysis of identifying linguistic phenomena for recognizing inference in text,","venue":null,"work_id":"d5ca1bb0-861e-4651-a212-c7920c7cf8b8","year":2014},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":85,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.819109Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:b7d32af75905f740efd83b2fd614e71bbc47678223ea376ad7f185ed07e4b97c","observation_id":"b17cb167-b484-401e-a616-35348e3c564f","resolution":{"observed_at":"2026-08-06T13:38:05.500337Z","resolver_source":"arxiv_id_nonexistent","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/k19-1033","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"EQUATE: A Benchmark Evaluation Framework for Quantitative Reasoning in Natural Language Inference,","venue":null,"work_id":"877a590d-b6cd-4a6a-b131-8a77480af1fd","year":2019},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":86,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.875892Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:e0e9ec1381424f403c50c960283b52ac9f3a6c49cedaeb1ad98c1f5c4121abc2","observation_id":"2351178e-b1e2-4ac3-aae2-2b68cae1df7e","resolution":{"observed_at":"2026-08-06T13:38:01.970328Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/p17-1075","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"Evaluation Metrics for Machine Reading Comprehension: Prerequisite Skills and Readability,","venue":null,"work_id":"5f7219ca-709c-40fb-87ab-760db5df550e","year":2017},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":87,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.932270Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:1ef92eb749cbbc4b637b850fa6bf963f410143d396bb3b205fb1605380d54b8d","observation_id":"40b6093e-fe1b-4e39-b590-8aa0b747a202","resolution":{"observed_at":"2026-08-06T13:38:01.777765Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:06.975742Z","title":"NaturalLI: Natural Logic Inference for Common Sense Reasoning,","venue":null,"work_id":"983b7ebc-d11b-4ca8-95cb-1e923135ee03","year":2014},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":88,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.991401Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:7d018939f7b1ff277ed4170cd5d6600854366b472e7d828cf2294f3717e11bc2","observation_id":"acffebb4-f9ff-4e31-b663-cfa23187988a","resolution":{"observed_at":"2026-08-06T13:38:07.125392Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.293552Z","title":"Building Textual Entailment Specialized Data Sets: a Methodology for Isolating Linguistic Phenomena Relevant to Inference.,","venue":null,"work_id":"81ba5cd1-7cda-4f15-9d93-deaf8637f64d","year":2010},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":89,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.733334Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:3b80c282883124567d60b030000f272d65dafa1811171057ed38a940eb5624c9","observation_id":"5dec987b-ab00-4b50-ba09-6af7a885574b","resolution":{"observed_at":"2026-08-06T13:38:07.450314Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2210.03256","last_updated":"2022-10-13T22:54:22Z","snapshot_observed_at":"2026-08-01T15:58:06.102904Z","submitted_at":"2022-10-06T23:39:01Z","title":"Not another Negation Benchmark: The NaN-NLI Test Suite for Sub-clausal Negation","version":2},"cited_work":{"arxiv_id":"2210.03256","doi":"10.48550/arxiv.2210.03256","metadata_source":"pith","pith_arxiv_id":"2210.03256","snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"Not another Negation Benchmark: The NaN-NLI Test Suite for Sub-clausal Negation","venue":"cs.CL","work_id":"c905e041-292b-4e3a-b290-3c0109784fb2","year":2022},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":90,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:01.153777Z"},"links":{"cited_paper":"/paper/2210.03256","citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:52341dedcce789b4c0e5b2fa1c41700f2d0a22e0011a423433b69a8f8b36f995","observation_id":"1106c391-1b24-4171-b2f6-dfb5fc5e9610","resolution":{"observed_at":"2026-08-06T13:38:01.441104Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:06.611565Z","title":"Neural Networks and Textual Inference: How did we get here and where do we go now?,","venue":null,"work_id":"1b5b9a4a-0c00-4d00-846b-99dcccb5dbc0","year":2017},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":91,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:01.264257Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:86cf09b1a2727f33fe4051c5c2fedb538b5033683e9756c68565dba334918469","observation_id":"c4943d82-5393-44fc-b3cd-d9f8ea6645d3","resolution":{"observed_at":"2026-08-06T13:38:06.760253Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/n18-2082","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"On the Evaluation of Semantic Phenomena in Neural Machine Translation Using Natural Language Inference,","venue":null,"work_id":"b9536499-dcd6-4300-a05a-7361790c33d4","year":2018},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":94,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:01.049741Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:451ccf342786fbd98007c011484cc3f7962a860940bd4b6a95396474cf019a9c","observation_id":"8fdf345c-55e3-41bd-8aec-bfd2f400b9e4","resolution":{"observed_at":"2026-08-06T13:38:01.605861Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:07.620554Z","title":"Available: https://aclanthology.org/2024.eacl-long.30/","venue":null,"work_id":"766e2bdf-923c-4aa7-be25-023f5e97628f","year":2024},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":520,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:59.366997Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:1c3dc2cd42072614ddc2083d49232563d55ef81065583f8221dfe098abdd772a","observation_id":"40049495-c160-45a7-b31a-d5732925ded0","resolution":{"observed_at":"2026-08-06T13:38:07.625044Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:57.304442Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":1422,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:57.304442Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:dab89dab098f8b043cf0dff92f5e4d38b416b9f2050aefee5e54ae06dea6b3af","observation_id":"066998a0-309a-4784-bc5e-efbdddc4b793","resolution":{"observed_at":"2026-08-06T13:37:57.304442Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:38:00.432949Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":4961,"source":"pdf_text","source_observed_at":"2026-08-06T13:38:00.432949Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:07f906c1971d0eb2b7643eff94bc0bfb6be87889e5009903ef35b249ef501d7f","observation_id":"f7cab572-fdec-491a-aa9c-c7017d1b647a","resolution":{"observed_at":"2026-08-06T13:38:00.432949Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T13:37:58.401646Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?","version":1},"reference_index":6018,"source":"pdf_text","source_observed_at":"2026-08-06T13:37:58.401646Z"},"links":{"citing_paper":"/paper/2507.20419"},"observation_digest":"sha256:7363fa5e113399922c9a6d442505703d87a525633448d4dae0af09b6f7b245d3","observation_id":"b815d477-6ea3-4d30-bf6b-0f182e0c0899","resolution":{"observed_at":"2026-08-06T13:37:58.401646Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2507.20419","last_updated":"2025-07-27T21:30:50Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-11T01:29:54.262386Z","submitted_at":"2025-07-27T21:30:50Z","title":"Survey of NLU Benchmarks Diagnosing Linguistic Phenomena: Why not Standardize Diagnostics Benchmarks?"},"reference_resolution":{"displayed":94,"state_counts":{"malformed_identifier":2,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":47,"verified_exact":20,"verified_fuzzy":25},"total_outbound_references":94},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"thesis":"As of 11 August 2026, this Paper Citation Record lists 94 of 94 outbound references and 0 inbound Pith citation observations for arXiv:2507.20419."}