{"as_of":"2026-08-13T11:59:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:e2feb7e5e5591426f0b87c15adcc75c819ec13b6e0f784dbf47c364e2cb8551d","coverage":[{"denominator":23,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":23,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T04:20:21.524087Z","state":"measured"},{"denominator":23,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":23,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-13T06:32:02.005865+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2412.01526/citation-record","integrity":"/paper/2412.01526/integrity","json":"/paper/2412.01526/citation-record.json","paper":"/paper/2412.01526"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:22.754914Z","title":"Large language models for software engineering: Sur- vey and open problems,","venue":null,"work_id":"4df335b9-b6fb-464b-8d77-5594a897f181","year":2023},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.426420Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:7b74bc4cf9b51ea247b4f9af53fc68ca226943c4dfa826cdab70a202ab0dd780","observation_id":"36a47dbc-3bde-46d7-bf17-f69ce5f5ccef","resolution":{"observed_at":"2026-08-12T04:20:22.759238Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:21.431387Z","title":"Software testing with large language models: Survey, landscape, and vision,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.431387Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:24941ae656b7fe8aca5233272ef91b49c43e872e36626985e7499b4526f27dfd","observation_id":"0dba7e71-d769-4c94-987a-01f9d7561e66","resolution":{"observed_at":"2026-08-12T04:20:21.431387Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:21.435724Z","title":"Large language models for software engineering: A systematic literature review,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.435724Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:92a933c7b831098557f05ec313fa052aa92a7f26e5ca31f5e5f916e125afce4b","observation_id":"0012f093-1f51-4be2-94c7-f904e3847ed0","resolution":{"observed_at":"2026-08-12T04:20:21.435724Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:21.440549Z","title":"A Survey on Evaluation of Large Language Models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.440549Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:a91d6a7a1bcaaf4bcf5ff8f9fa95786d0d509456020af70aa886e92149e96a07","observation_id":"f48b6488-b426-477f-bc63-905a50902254","resolution":{"observed_at":"2026-08-12T04:20:21.440549Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.08216","last_updated":"2024-06-12T13:45:45Z","snapshot_observed_at":"2026-08-12T23:44:33.558286Z","submitted_at":"2024-06-12T13:45:45Z","title":"A Software Engineering Perspective on Testing Large Language Models: Research, Practice, Tools and Benchmarks","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.08216","snapshot_observed_at":"2026-08-12T04:20:21.445014Z","title":"A Software Engineering Perspective on Testing Large Language Models: Research, Practice, Tools and Benchmarks,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.445014Z"},"links":{"cited_paper":"/paper/2406.08216","citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:0acc8019c7da450cb96dd2deed076e245f261a2e9f6e498780b67c7ecca2971b","observation_id":"068aba4a-d1a9-4192-8403-7350dcc76267","resolution":{"observed_at":"2026-08-12T04:20:21.445014Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T11:58:24.516369Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-12T04:20:21.449441Z","title":"Evaluating large language models trained on code,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.449441Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:2a53011d19895f465ad6ea18677894fcabec5248a468a75538584277debc1ecc","observation_id":"4555590f-4a22-452e-84b2-8e682f5bb20f","resolution":{"observed_at":"2026-08-12T04:20:21.449441Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:21.454253Z","title":"Codereval: A benchmark of pragmatic code generation with generative pre-trained models,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.454253Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:3a1203308ad11570d613006334c6669a3b81299bec0857babaab47c5bcbf9328","observation_id":"236712a5-0d82-495a-a02b-1859d596513c","resolution":{"observed_at":"2026-08-12T04:20:21.454253Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:21.464228Z","title":"Towards standarized benchmarks of LLMs in software modeling tasks: a conceptual framework,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.464228Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:05757eb7495f58f7f7449b3d0c012c64defe213cda51141c2b0a73dceb58932a","observation_id":"676a281d-38e1-4338-a028-8d24e3328ef6","resolution":{"observed_at":"2026-08-12T04:20:21.464228Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:22.459777Z","title":"Using benchmarking infrastructure to evaluate llm performance on cs concept inventories: Challenges, opportunities, and critiques,","venue":null,"work_id":"f723ae7d-ee75-4bf2-86d4-473908cddb5e","year":2024},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.468618Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:286b30b7c26790eec021c5a6414e2c4848cbf65d2f357a0512a44d315af55858","observation_id":"2b3d364f-f575-4ba1-9924-bdbbe3c5d0ad","resolution":{"observed_at":"2026-08-12T04:20:22.463999Z","resolver_source":"arxiv_id_nonexistent","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:22.281253Z","title":"Condefects: A complementary dataset to address the data leakage concern for llm-based fault localization and program repair,","venue":null,"work_id":"d2216e6d-bd1e-47d8-9f82-66cb551bb6d4","year":2024},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.472665Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:c6a7876a9197149aa4f4725e631c70c6b74c4b518e0c9b8e19cd3ec761489850","observation_id":"149a2248-9216-4ffd-8f0b-ba279108ddea","resolution":{"observed_at":"2026-08-12T04:20:22.285490Z","resolver_source":"arxiv_id_nonexistent","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:22.092490Z","title":"BOLD: Dataset and Metrics for Measuring Biases in Open-Ended Language Generation,","venue":null,"work_id":"b6ee810e-fa7a-4906-938c-5a090424a031","year":2021},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.476747Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:6dbe3ab1f519d83d45480f24e166712d02f71a9410db1e86ece58d2909353ac5","observation_id":"d67b6f95-605e-4ca9-836a-4789e64b2353","resolution":{"observed_at":"2026-08-12T04:20:22.096588Z","resolver_source":"arxiv_id_nonexistent","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T04:20:21.483043Z","title":"Holistic Evaluation of Language Models,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.483043Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:f8187751b0aa9a819ef4e109ad1bfc35d5235826844596174b171e5a34a77035","observation_id":"ffd2df8d-b59f-4695-9cc1-eb5b565deece","resolution":{"observed_at":"2026-08-12T04:20:21.483043Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.01964","last_updated":"2023-11-03T14:59:54Z","snapshot_observed_at":"2026-08-13T05:33:22.686364Z","submitted_at":"2023-11-03T14:59:54Z","title":"Don't Make Your LLM an Evaluation Benchmark Cheater","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.01964","snapshot_observed_at":"2026-08-12T04:20:21.487565Z","title":"Don’t Make Your LLM an Evaluation Benchmark Cheater,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.487565Z"},"links":{"cited_paper":"/paper/2311.01964","citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:8af8e1725e115c70f1b0622f340e877a7d4404cffce679628769d816f39c881e","observation_id":"5e8413bb-b11a-41b9-b492-e82ad20b5500","resolution":{"observed_at":"2026-08-12T04:20:21.487565Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:22.725796Z","title":"NLP evaluation in trouble: On the need to measure LLM data contamination for each benchmark,","venue":null,"work_id":"050a6f86-989f-4f02-ab32-e58d330fa1ce","year":2023},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.491898Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:c9968879ffcd7a4ec993223e70470d61975e67e79e5201a5210d16daec343b65","observation_id":"06b07fee-2001-47b6-9caf-33dbff040a34","resolution":{"observed_at":"2026-08-12T04:20:22.730332Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:22.713619Z","title":"Task Contamination: Language Models May Not Be Few-Shot Anymore,","venue":null,"work_id":"44a88bd9-9fb6-4cd0-beda-3b587cb25796","year":2024},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.495785Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:9843ea77823d3f2dfcf5abf7f49a6756f3deed9532b6c46130df40207e911558","observation_id":"d241f699-5b41-4b59-9583-b9b5290bfc30","resolution":{"observed_at":"2026-08-12T04:20:22.717836Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:22.701087Z","title":"A test generation strategy for pairwise testing,","venue":null,"work_id":"6ae7c997-2c09-4789-bf6a-458ffa6d57f4","year":2002},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.499796Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:eaf2c97ee40c0d5df13b14a90d929e6797ed605064b595d8625ef43191559962","observation_id":"06bc2373-2751-4211-9c62-f2b9a7b2ea26","resolution":{"observed_at":"2026-08-12T04:20:22.705319Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:22.688457Z","title":"Combinatorial test design in practice,","venue":null,"work_id":"f38032ef-310c-416a-8e10-619697e2c825","year":2010},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.504311Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:e50e022fdbc6d1334a9043bdd4248021ab976c170dd8d80ca25406dc4177f595","observation_id":"42dce237-fb1f-419d-beb3-d164c71fa7ce","resolution":{"observed_at":"2026-08-12T04:20:22.692897Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"document/1201189","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:21.919611Z","title":"Using benchmarking to advance research: a challenge to software engineering,","venue":null,"work_id":"c71f15b1-b694-40de-a1dc-6788907e4fb1","year":2003},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.508384Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:46b862fba4e6ec1724b3d043a1feb24bc5d4663355d23c4dbfbc6bd561c76e42","observation_id":"9944ad66-216d-4120-8a31-196d1054daef","resolution":{"observed_at":"2026-08-12T04:20:21.926313Z","resolver_source":"raw_fallback","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:21.783645Z","title":"Using combinatorial benchmark construction to improve the assessment of concurrency bug detection tools,","venue":null,"work_id":"45241ec7-2126-4f5b-aafd-8799220eff71","year":2012},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.512271Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:848c1126fdd7e0272ac320e35db0292066147e79d8fdbe4466589dc77f161617","observation_id":"babcd247-2587-4e4a-bdec-24a7767e85fd","resolution":{"observed_at":"2026-08-12T04:20:21.789763Z","resolver_source":"arxiv_id_nonexistent","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:22.674806Z","title":"Applying pairwise combinatorial testing to large language model testing,","venue":null,"work_id":"5ffebf4c-a709-477f-8c7b-6e662b945b61","year":2023},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.516299Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:688f159fb5158cf5f858ab78e35b94b0d0aa83ddb2bf255d2806572a76c2d506","observation_id":"49c0fa33-9bfe-4b02-a60e-1b8da0d396f9","resolution":{"observed_at":"2026-08-12T04:20:22.679251Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.11443","last_updated":"2024-02-18T03:40:06Z","snapshot_observed_at":"2026-08-13T04:16:52.184675Z","submitted_at":"2024-02-18T03:40:06Z","title":"Benchmark Self-Evolving: A Multi-Agent Framework for Dynamic LLM Evaluation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.11443","snapshot_observed_at":"2026-08-12T04:20:21.520097Z","title":"Benchmark Self- Evolving: A Multi-Agent Framework for Dynamic LLM Evaluation,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.520097Z"},"links":{"cited_paper":"/paper/2402.11443","citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:b0ca93b79d7373196d2c7497935df98d9771f0bb00f7158a418aef3be3e61245","observation_id":"a62219f4-63e0-4c44-b634-98e80ad385fb","resolution":{"observed_at":"2026-08-12T04:20:21.520097Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.14865","last_updated":"2024-06-07T09:19:45Z","snapshot_observed_at":"2026-08-13T04:13:58.496742Z","submitted_at":"2024-02-21T06:46:34Z","title":"Dynamic Evaluation of Large Language Models by Meta Probing Agents","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.14865","snapshot_observed_at":"2026-08-12T04:20:21.524087Z","title":"Dynamic Evaluation of Large Language Models by Meta Probing Agents,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.524087Z"},"links":{"cited_paper":"/paper/2402.14865","citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:86fca7f6cfbcc635543de840b4a720b18f0de6f2899dc2b5f0157d1571ed74cb","observation_id":"41996491-cd86-4db3-8569-cc51fdad9ac3","resolution":{"observed_at":"2026-08-12T04:20:21.524087Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T04:20:22.638304Z","title":"Available: https://doi.org/10.1145/3597503.3623316","venue":null,"work_id":"a39e0813-b57b-4a2d-b384-9251eb1bc0d3","year":null},"citing_paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design","version":1},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-12T04:20:21.459075Z"},"links":{"citing_paper":"/paper/2412.01526"},"observation_digest":"sha256:647b2a12b3664371226902f4b55185d5b2f4689b7f8e3a2098e9018218e3cfe4","observation_id":"8e08a151-0be0-48f5-9029-1100fa38c204","resolution":{"observed_at":"2026-08-12T04:20:22.642217Z","resolver_source":"arxiv_id_nonexistent","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2412.01526","last_updated":"2024-12-02T14:18:32Z","latest_version":1,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-12T18:25:11.618927Z","submitted_at":"2024-12-02T14:18:32Z","title":"Addressing Data Leakage in HumanEval Using Combinatorial Test Design"},"reference_resolution":{"displayed":23,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":11,"verified_exact":6,"verified_fuzzy":6},"total_outbound_references":23},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"thesis":"As of 13 August 2026, this Paper Citation Record lists 23 of 23 outbound references and 0 inbound Pith citation observations for arXiv:2412.01526."}