{"as_of":"2026-08-17T17:12:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:c38173b4432b9f0caa8cec55d8b4de0f466a67b4441dace73e1778b4c87d4af3","coverage":[{"denominator":80,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":80,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T11:30:27.995986Z","state":"measured"},{"denominator":80,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":80,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-17T06:30:58.91139+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2411.18151/citation-record","integrity":"/paper/2411.18151/integrity","json":"/paper/2411.18151/citation-record.json","paper":"/paper/2411.18151"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.580872Z","title":null,"venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.580872Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:00d94c348b132ed96a245dbe14035316616be4a7783481eebf4ba222c86ba7d2","observation_id":"e2887da4-0e8c-4225-873a-7ab27235200e","resolution":{"observed_at":"2026-08-12T11:30:27.580872Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1145/3377814","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:28.137187Z","title":"Ahmed, Nisheeth Srivastava, Renuka Sindhgatta, and Amey Karkare","venue":null,"work_id":"1ddc47a7-1bdb-4fff-ba98-1271373d77a8","year":2020},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.586029Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:4897322708b17ea7d063d0d66a23712df495abc38d7986e08fd5c58b12d8ce86","observation_id":"e033df61-614f-4bae-8ae4-26bf9e84f9c7","resolution":{"observed_at":"2026-08-12T11:30:28.142580Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.21061/ctete-rms.v1.c.1","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:28.119624Z","title":null,"venue":null,"work_id":"5560b2bc-605f-488b-8f04-8771d656d6f4","year":2018},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.591276Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:817b547f9eed1f9a1604b295c33767b529294b5d0fd80591d7f7afc4571ce576","observation_id":"b075a8eb-7cdf-42bc-8d20-14a63aa85539","resolution":{"observed_at":"2026-08-12T11:30:28.125581Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.595937Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.595937Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:35ecf9014dae2653c900295e92dcaa344d14b091277c88a1415b8c4ea8337cf2","observation_id":"5a58db14-67d7-4581-b011-810456b066c9","resolution":{"observed_at":"2026-08-12T11:30:27.595937Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.600605Z","title":null,"venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.600605Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:6919dc940da13f9d09d709766c40606df7fd1cfa23f41ca6cd90753aaa971243","observation_id":"fdc7d5e0-257f-49db-ad7c-66497dfaf47e","resolution":{"observed_at":"2026-08-12T11:30:27.600605Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"0969.36111","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.328268Z","title":null,"venue":null,"work_id":"5aff724e-e2dc-4337-acf8-bfa677f2666a","year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.605855Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:572e6bef7f1d585d2bdc6c51ed5b187866a2879c6cfe08e90c132462ae1dfc5f","observation_id":"b8558598-1afb-4892-b264-3157c3d48771","resolution":{"observed_at":"2026-08-12T11:30:32.335459Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.610863Z","title":null,"venue":null,"work_id":null,"year":2014},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.610863Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:df735fe9ccfdc1c9e34f45648d5cefe8929211d186bb17541cfe1987369e73ce","observation_id":"0f300b67-252e-40b7-bb81-2847ecc1b5ad","resolution":{"observed_at":"2026-08-12T11:30:27.610863Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.615271Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.615271Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:a85fcde06e84d8de5dfa617df6e1acbbbd77eace2006f2cc9e64282131aa77b1","observation_id":"350c4baa-de3a-4bcb-9fa5-bd153a69af6d","resolution":{"observed_at":"2026-08-12T11:30:27.615271Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.619568Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.619568Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:45fd616a788d9819d13f8387e4ea5a1f4fd47af177adcd53750ef474db5237ac","observation_id":"ae96dc2c-639e-4e17-8a58-d517c3b1a528","resolution":{"observed_at":"2026-08-12T11:30:27.619568Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.623788Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.623788Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:77dc5abdc11875e55bc94e851cb299016c25f87c68a5eb62b3c16157054b001b","observation_id":"63b06e05-c23c-4c36-85b6-cef3bb8cb188","resolution":{"observed_at":"2026-08-12T11:30:27.623788Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.641488Z","title":"Becker, James Finnie-Ansley, Arto Hellas, Juho Leinonen, Andrew Luxton-Reilly, Brent N","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.641488Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:b20aa8be372490e81e20c66bc21aae1dc7fb7f8fa0d1eee5e77f5cd1de5b58f4","observation_id":"ebf653e9-3bd1-4ff0-a598-12576604280c","resolution":{"observed_at":"2026-08-12T11:30:27.641488Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.732430Z","title":"Becker, Catherine Mooney, John Homer, Zachary C Albrecht, and Garrett B","venue":null,"work_id":"9dd434f7-945c-47c6-abee-68a1fa400b5c","year":null},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.646147Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:2b6cd3cfa61c479322035849faea67cc804691ebcde17039da6b19458299b546","observation_id":"55428469-3402-4099-bd03-d7af87c2aafb","resolution":{"observed_at":"2026-08-12T11:30:32.737238Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"1276.23612","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:31.677141Z","title":"Falkner and Katrina E","venue":null,"work_id":"ab2d5b62-60cd-40c0-ab7c-7e7c954e5529","year":2012},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.656180Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:b8b00c1dfc8cef8b5a433f1dcae2cb368ead13282325d100976601e3ab6292c1","observation_id":"3259f061-7343-4314-96dc-496a11cf7193","resolution":{"observed_at":"2026-08-12T11:30:31.684371Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.660861Z","title":"Fiannaca, Chinmay Kulkarni, Carrie J Cai, and Michael Terry","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.660861Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:5a3ba8217b9ad0daa0b10341427034570b1915d9f43c1dbf05828b48c72e79f2","observation_id":"128bf2ba-094d-4fdc-b5d6-edb88a95ae69","resolution":{"observed_at":"2026-08-12T11:30:27.660861Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.717292Z","title":null,"venue":null,"work_id":"9179dfae-5daf-4b7c-9e40-cf9884a4e261","year":2005},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.666187Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:5d30c51913ed2f24b9ceb8ddee9900f65b8eb28f80f99c0e83e07b8451dd3507","observation_id":"502daedb-2b28-4276-b7c3-b7cf731e3e37","resolution":{"observed_at":"2026-08-12T11:30:32.722233Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.701808Z","title":null,"venue":null,"work_id":"f80cd2e4-7192-4047-89af-0919695df2fa","year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.670854Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:8008d4c470eb6b6ce8d3e9d7e8c0a290972567d0804ed922734f37994ab1dcaf","observation_id":"c70e0d49-eeaa-4d85-a027-0c47fe048d28","resolution":{"observed_at":"2026-08-12T11:30:32.707092Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.675508Z","title":"Glassman, Aaron Lin, Carrie J","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.675508Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:4385e92c949b2b4a7cf1b419f3abaaf6c7c1e223e5a614f7bd6d9e4e3306a8f2","observation_id":"caa45737-8407-4d33-97f8-9dccdafb5fb8","resolution":{"observed_at":"2026-08-12T11:30:27.675508Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"6527.34067","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:31.441741Z","title":"Guo, Julia M","venue":null,"work_id":"f0e08905-db5d-4f7b-af01-5dca52a2877e","year":2020},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.680369Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:94c04aac81818119ceede87c6cae40090828a63afd04f1d7685f74874df24ce7","observation_id":"a42b8d7d-b2aa-4466-bbee-f5a52910584a","resolution":{"observed_at":"2026-08-12T11:30:31.450088Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2017.81204","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:31.336861Z","title":null,"venue":null,"work_id":"8ac92cdd-5061-46d9-b147-a73d289de4cb","year":2017},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.684804Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:e1c49e81b1bfad2d44045af956c8e7cbce8f071081812f802d4e988b8c8bf65f","observation_id":"e01261a7-c138-4f4f-839e-904df9e79271","resolution":{"observed_at":"2026-08-12T11:30:31.346742Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"7091.32058","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:31.221222Z","title":"Ajanovski, Mirela Gutica, Timo Hynninen, Antti Knutas, Juho Leinonen, Chris Messom, and Soohyun Nam Liao","venue":null,"work_id":"47f19c26-b8df-49fd-8bd1-eb94403bfeb2","year":2018},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.689558Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:e68c27041b22eb425e558371606aaffad4d2182f38ea00393c2a8ca7204a810d","observation_id":"b57ce87c-0a21-4a59-8811-adcab921a9da","resolution":{"observed_at":"2026-08-12T11:30:31.231311Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.686460Z","title":"2024.Predicting Results of Social Science Experiments Using Large Language Models","venue":null,"work_id":"4593df07-71d3-4cf7-bd47-78e6027b5408","year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.699118Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:6f3c489e5442cff0f7783c4ee6e6ca1ddcd295395e9ee698c62760c1733bff7a","observation_id":"3cf5202d-8683-4f2a-89c0-d20b34387f2b","resolution":{"observed_at":"2026-08-12T11:30:32.691282Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.671714Z","title":null,"venue":null,"work_id":"e53e07ad-fdb1-4cb1-a609-9c9f490216a5","year":1995},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.703538Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:d88235cc67b39bf07106efe24ea27268df1d1a206f67f659ac6a999cb895de21","observation_id":"78d8315f-0ea7-4c15-b9cd-2a016d47304d","resolution":{"observed_at":"2026-08-12T11:30:32.676204Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.707850Z","title":"Hutto and Eric Gilbert","venue":null,"work_id":null,"year":2014},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.707850Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:c432b9349f3765e1a095b1d4f448cfb7fe1e58565e30c2517995d834ce9f6426","observation_id":"cdafeb7c-8f4a-496c-aee6-8cb4be7beb70","resolution":{"observed_at":"2026-08-12T11:30:27.707850Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.712309Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.712309Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:7cb26330aac64770c80032002ce5aebf01a937201b4bcf56581f471792f86384","observation_id":"ab92ac17-8e2c-455e-b2c4-6a6d273ead20","resolution":{"observed_at":"2026-08-12T11:30:27.712309Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.717800Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.717800Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:fefb70378216efd185d7632535541ec7608df9ee14f8b5b98cd0778f94b71a10","observation_id":"54d4dcaa-9036-4e64-b034-0b4559baec24","resolution":{"observed_at":"2026-08-12T11:30:27.717800Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2023.22422","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:30.841156Z","title":null,"venue":null,"work_id":"4ee87397-795f-4721-b3ba-556298ae8bfa","year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.722750Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:03108e36a147166205a6020a928d40d6830b68262a022c0f88c5ba4e9eddf345","observation_id":"1cf38f08-12a7-4418-a77d-f6a4a0447241","resolution":{"observed_at":"2026-08-12T11:30:30.849444Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2309.10694","last_updated":"2023-09-30T07:22:41Z","snapshot_observed_at":"2026-08-16T14:59:23.982133Z","submitted_at":"2023-09-19T15:29:12Z","title":"\"With Great Power Comes Great Responsibility!\": Student and Instructor Perspectives on the influence of LLMs on Undergraduate Engineering Education","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.10694","snapshot_observed_at":"2026-08-12T11:30:27.727526Z","title":"With Great Power Comes Great Respon- sibility!","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.727526Z"},"links":{"cited_paper":"/paper/2309.10694","citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:6a19250b0936cddb7f38b5d2e1e9644345ddf70dabd77ac9d23313fa32a25d06","observation_id":"e8756c65-a449-4b04-a586-75a1193ad08e","resolution":{"observed_at":"2026-08-12T11:30:27.727526Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.732282Z","title":"Ericson, David Weintrop, and Tovi Grossman","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.732282Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:7ab3a48fca4282945ad83cb01eb93f243ca680b8040114edf506cf345a251dbf","observation_id":"dc9ab964-d753-4f28-a6ce-6f00f669f372","resolution":{"observed_at":"2026-08-12T11:30:27.732282Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.736826Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.736826Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:ecb184cb344011af3270e590c234b112e99255adf93e93ac8ce922e3736c7b32","observation_id":"260a696e-7097-4549-b6e3-1c85fcc01c95","resolution":{"observed_at":"2026-08-12T11:30:27.736826Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.741550Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.741550Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:b03d4101bd6e2bd891a24be1027023e831a849dffc44183e55c6b7bc853dc9c9","observation_id":"283b7718-1bb0-494a-b123-b1d5e8ad5702","resolution":{"observed_at":"2026-08-12T11:30:27.741550Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.746218Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.746218Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:6e1331d2fc727e9c37d65c0182210d1e8e6f2b43cd089585c12babeeebcf7123","observation_id":"258e90de-da70-4024-8282-cd671bf3f74a","resolution":{"observed_at":"2026-08-12T11:30:27.746218Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.08572","last_updated":"2023-08-15T19:48:31Z","snapshot_observed_at":"2026-08-16T15:08:15.938364Z","submitted_at":"2023-08-15T19:48:31Z","title":"Large Language Models in Introductory Programming Education: ChatGPT's Performance and Implications for Assessments","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.08572","snapshot_observed_at":"2026-08-12T11:30:27.750926Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.750926Z"},"links":{"cited_paper":"/paper/2308.08572","citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:58ab785f9febe6a3610002cd8d0dc6057071b318b441092e5b73b429ba5213bd","observation_id":"7e4ceb67-f775-4ca6-902f-c7eec62d2c95","resolution":{"observed_at":"2026-08-12T11:30:27.750926Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.756198Z","title":null,"venue":null,"work_id":null,"year":1975},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.756198Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:4787d1ee0972af31045e1c43fec114c33d63b91d71b3b13e08c5715427456aae","observation_id":"8efe8acc-1dfc-4ae9-8a68-458dec23b7da","resolution":{"observed_at":"2026-08-12T11:30:27.756198Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.645655Z","title":null,"venue":null,"work_id":"91fdef8a-a8fd-4cfc-a1cf-74062fb8f14e","year":2016},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.761650Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:8c928e5cd9a11368d5ac3c063eba39ce23c0d3924b7326ee4bbaa0a3ac8f1754","observation_id":"e3ab300c-637e-4b94-954d-358570a3bfb3","resolution":{"observed_at":"2026-08-12T11:30:32.651783Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.04873","last_updated":"2024-11-22T01:13:13Z","snapshot_observed_at":"2026-08-16T13:36:39.032453Z","submitted_at":"2024-07-05T21:44:11Z","title":"Evaluating Language Models for Generating and Judging Programming Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.04873","snapshot_observed_at":"2026-08-12T11:30:27.766081Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.766081Z"},"links":{"cited_paper":"/paper/2407.04873","citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:df02a9a91c1000fa123a3f2126fe52991ee48a39bde0a7882d5e5633d0b85792","observation_id":"3e3600b4-774a-474f-a049-d7c17480b0ba","resolution":{"observed_at":"2026-08-12T11:30:27.766081Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.776293Z","title":"Ban It Till We Understand It","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.776293Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:8f7ef9ac8e21eb86738973d80b6cd7b1f5792327677c0f11e1ebd235b4713d87","observation_id":"7767876f-8d09-4983-b4b4-8962a11064ab","resolution":{"observed_at":"2026-08-12T11:30:27.776293Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.631572Z","title":null,"venue":null,"work_id":"caec3336-dff5-43c7-a0d1-98313031496a","year":null},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.780949Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:8bdedb33fb298c9687a0d2032d6739ec183427af740c273059ce40c39221f35c","observation_id":"002a46f1-21fa-498d-86cb-dc9b82ac1149","resolution":{"observed_at":"2026-08-12T11:30:32.636005Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.794774Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.794774Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:b029ce5e3bc6a06b7c8553142248e0280c33d843412f2a7e3671ac86a2e37dbb","observation_id":"97be0ac3-4651-4154-81da-87984be30248","resolution":{"observed_at":"2026-08-12T11:30:27.794774Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.617192Z","title":null,"venue":null,"work_id":"15ddda4d-808f-4b71-b419-a40630f5bace","year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.799392Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:f9cdc6bb782c1d3611c10a49eda258d0221813a2490ca4563d91cb2a7b848f9d","observation_id":"39c78a17-9e8a-4fb0-8e8e-2ee2c7dcc839","resolution":{"observed_at":"2026-08-12T11:30:32.621703Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.804217Z","title":"Let Them Try to Figure It Out First","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.804217Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:6a87594ba42a496a67fc8b1392aad78c17fa815df3191979b54dc647b9bb62ee","observation_id":"2aeb9b1e-3827-4788-be03-cf27effe1a62","resolution":{"observed_at":"2026-08-12T11:30:27.804217Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.601543Z","title":null,"venue":null,"work_id":"5abc755b-0c9d-44bb-bf72-b53613fe46fa","year":null},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.808450Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:75c3ecf6278abc4a1d84137c95c577886d9808f5b2db82e84ca451d5ef4576f8","observation_id":"845fbdc3-ecda-48b5-98fd-ad8163b6b065","resolution":{"observed_at":"2026-08-12T11:30:32.606679Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"0969.36109","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:30.130170Z","title":null,"venue":null,"work_id":"d143488a-07cb-4f1f-9650-7d6fa7955d61","year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.818467Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:270f40ecfe3f51173ca5200990283a2f48a5b4dd5ae0ab3d04ef3a143dacddb5","observation_id":"d6c88357-5e80-47fd-a947-3510e34f444d","resolution":{"observed_at":"2026-08-12T11:30:30.138032Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.585837Z","title":"Ok Pal, We Have to Code That Now","venue":null,"work_id":"0c074c91-aeda-4cdc-9a85-1b588fb4c408","year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.824069Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:d4bd43b4ab933ab36f817bce1f93b236317026861b8738cb85e60ef253443075","observation_id":"60ca70de-188f-4fa2-a7de-b4e04d89ef65","resolution":{"observed_at":"2026-08-12T11:30:32.590967Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.813325Z","title":"In Proceedings of the 54th ACM Technical Symposium on Computer Science Education V","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.813325Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:66d38f098d6003d55af23c18ed2cd456accce8356237ffff8b7d4b19b52c44a7","observation_id":"8573eb57-3a88-44c3-ba0e-f5cf12236af9","resolution":{"observed_at":"2026-08-12T11:30:27.813325Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.835342Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.835342Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:1768a3039f23f21512f894fb3a5b02e8ce5e48c838f6b762ad723724acec03be","observation_id":"5d120c56-646c-4bf8-8e11-4c44f3475c3f","resolution":{"observed_at":"2026-08-12T11:30:27.835342Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.840475Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.840475Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:3109d7b95b4538c86f8c9d0dab9ce3a165eddb44561a70985d28f34e6b5261b9","observation_id":"c85326dc-50bb-4107-8d9e-6dc9fe4782c7","resolution":{"observed_at":"2026-08-12T11:30:27.840475Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.829681Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.829681Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:e1f0574ad98305485d71196659fe82696b773348792ff74ee8cf8a25337c2d07","observation_id":"3de0fd73-dd4a-4f56-a354-6b9e254348ac","resolution":{"observed_at":"2026-08-12T11:30:27.829681Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.570405Z","title":null,"venue":null,"work_id":"f3b206a7-bd38-4d43-8f31-015df5a270d9","year":2015},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.850832Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:63fb8164ef81b6de266b16a888c7973b9b44d760b420a97fabf74fa9b8f949fc","observation_id":"9d4c9aa0-d5b2-4dae-8017-f10fd8a97eed","resolution":{"observed_at":"2026-08-12T11:30:32.575663Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.855457Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.855457Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:490fb16bc714751ffcfa2fdc428812a9627a90f0c5cf656c19f18cc9c8ea93db","observation_id":"b55b55f6-1809-4a24-987c-0b8b20bd7dec","resolution":{"observed_at":"2026-08-12T11:30:27.855457Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.845664Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.845664Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:ab3d6d14b99f8d908aafcd693bf9c8b97766c6f285a3821647cbce07f1d44d98","observation_id":"0badc109-39ff-4ace-8cfb-2f2de304ea8d","resolution":{"observed_at":"2026-08-12T11:30:27.845664Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.864974Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.864974Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:5d56e9f6711b8f836169b8c5d43c0812e759f2dad4d525f0fe5b91765861f0b8","observation_id":"fad38ca3-fc09-4220-af08-a45df4a1ba42","resolution":{"observed_at":"2026-08-12T11:30:27.864974Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.869591Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.869591Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:b6c978c4c3f8b3e7286a01e4771a40f53b12f18479cd0e5e5e55eb1257413b16","observation_id":"ba92dd5f-86af-4eae-a5dc-2be776d8b849","resolution":{"observed_at":"2026-08-12T11:30:27.869591Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.860474Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.860474Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:2c7ab0232137e53352802524bc718e357ccef983e9b9ffb669a4e8367070e0ce","observation_id":"8aca14a4-5e93-45ba-a183-18479f1f07ef","resolution":{"observed_at":"2026-08-12T11:30:27.860474Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.878783Z","title":null,"venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.878783Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:9297fa2cff9f6911a22c71257b2c7e010ca0e48cc25847aee7d281a258842041","observation_id":"9b5f3ed2-99e5-46e9-822c-6184844171d9","resolution":{"observed_at":"2026-08-12T11:30:27.878783Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2012.66535","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:29.446352Z","title":null,"venue":null,"work_id":"deed239f-ed75-48cc-aaf8-85c1580bb154","year":2012},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.883859Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:2f7d24ab8e712cb6f9e3ad424adba6a43692cda5579ed641f5f4a6c5940f4cae","observation_id":"8a11d673-7fa8-47b8-966d-be0274a97459","resolution":{"observed_at":"2026-08-12T11:30:29.453686Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.874527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.874527Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:ad86146c4410f8e9335d6706f10cdc943efff19f4b044637884c5c360e30da18","observation_id":"29d41e53-a56d-45ec-a108-7928ef136fc8","resolution":{"observed_at":"2026-08-12T11:30:27.874527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.554027Z","title":"Becker, Robert Nix, Brent N","venue":null,"work_id":"3c5c486c-a034-4d43-9577-916cef46ee63","year":null},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.893325Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:4788b371465b57ad7c33812c9b4b27e9a7e4142b153ca7e184c0b5e83e7b8ea4","observation_id":"5e096b9a-ec56-4610-a6dc-ae30737fb2dc","resolution":{"observed_at":"2026-08-12T11:30:32.559852Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"0977.32309","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:29.287469Z","title":null,"venue":null,"work_id":"2cc06ab7-bc84-47db-92d1-b0700a9a11f0","year":2018},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":64,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.903223Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:c971a15570c639f082d2fb9273be119ae76206f9b9601e24c88f52af912f506d","observation_id":"520347c2-8c49-41ec-8ea9-2024fa994ae2","resolution":{"observed_at":"2026-08-12T11:30:29.294844Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.888690Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":65,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.888690Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:02aa55657ab78403c947bffe1c7a7fae012413a59f59a22bf0713b4bc498fa2d","observation_id":"6b3afdcd-63a0-4597-b997-233068345fbf","resolution":{"observed_at":"2026-08-12T11:30:27.888690Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.913197Z","title":"It’s Weird That it Knows What I Want","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":66,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.913197Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:4ad14e02a9fedf86f64e63b82b552ef22f082449a5fe7876db36e5d9678da049","observation_id":"642e95e8-e8c7-4005-9495-2d948719d26b","resolution":{"observed_at":"2026-08-12T11:30:27.913197Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.898445Z","title":"In Proceedings of the 54th ACM Technical Symposium on Computer Science Education V","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":67,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.898445Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:69fb19a4479177226137ed5d125d518bac188957bb939ccc76a2f1976bdb1e76","observation_id":"0762d504-54d2-4793-bc71-6d97e968ad95","resolution":{"observed_at":"2026-08-12T11:30:27.898445Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.923300Z","title":"Price, Samiha Marwan, and Joseph Jay Williams","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":68,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.923300Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:de7d40ae18e0693c39827f18a0249eca921b97a6f446ddef0f9a27068ca3dc26","observation_id":"4b879563-1418-48b4-a809-15bc9eeb3355","resolution":{"observed_at":"2026-08-12T11:30:27.923300Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.908275Z","title":null,"venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":69,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.908275Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:8cac3149e3da3215ad03d592760f9846062401503d91635eff62b81a5506b9e5","observation_id":"3cd10ca0-a1e4-4964-8bb5-3e01903cf36d","resolution":{"observed_at":"2026-08-12T11:30:27.908275Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"1833.35618","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:28.961268Z","title":null,"venue":null,"work_id":"05cc0cb9-5224-47d7-b17f-9aeb7af5b40a","year":2022},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":70,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.933509Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:fc64da21d3f7a4abb9990260a040ae6c41fc8a1d80a1d651586a8031d3b3178f","observation_id":"55544338-25ad-4635-a40f-a4f12ea0886f","resolution":{"observed_at":"2026-08-12T11:30:28.969555Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.918367Z","title":"Becker, Bailey Kimmel, Jared Wright, and Ben Briggs","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":71,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.918367Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:3c2ab75362f7058c9d4f4561c79c0358db4dbec524b22336efa29679586f02ba","observation_id":"2c8d6d53-c0f4-482a-8f7f-db8b4782cc55","resolution":{"observed_at":"2026-08-12T11:30:27.918367Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.539670Z","title":"Price, Samiha Marwan, Michael Winters, and Joseph Jay Williams","venue":null,"work_id":"985d4ccc-cd32-47c8-826a-38c40be79bf4","year":2020},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":73,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.928074Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:94a72eaa0320d7590b238972f8eb97e029ea8e650573c7e37f982b01518b007e","observation_id":"ed01883d-9245-4224-a096-196fe164d436","resolution":{"observed_at":"2026-08-12T11:30:32.544289Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.20792","last_updated":"2024-07-30T12:55:42Z","snapshot_observed_at":"2026-08-16T13:29:53.313637Z","submitted_at":"2024-07-30T12:55:42Z","title":"How Novice Programmers Use and Experience ChatGPT when Solving Programming Exercises in an Introductory Course","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.20792","snapshot_observed_at":"2026-08-12T11:30:27.952170Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":74,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.952170Z"},"links":{"cited_paper":"/paper/2407.20792","citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:c028f9068b6a707698e845ed03ae9084655dfc1c036da4fb1f0bcc30f3f9bd60","observation_id":"aa3f8c6c-f504-4640-baa6-eee803b0b1de","resolution":{"observed_at":"2026-08-12T11:30:27.952170Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:32.524763Z","title":null,"venue":null,"work_id":"bad5133b-a824-4748-9377-d0b311c03ace","year":2006},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":75,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.938338Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:78fe4bc47e3cab5884112d0979f38430824b00ee0490acb7b5e25cc2ee4f8953","observation_id":"1bf578ff-6020-4ba7-b3d2-5164531dd5d3","resolution":{"observed_at":"2026-08-12T11:30:32.530048Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.963619Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":76,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.963619Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:6e94d0837b0e93c7cfb24f11151b3d47f9289c0be8705ddf18dd6a23d9fd0565","observation_id":"cb53221f-2e75-4dac-91b8-951000c20b2a","resolution":{"observed_at":"2026-08-12T11:30:27.963619Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2019.16020","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:28.806070Z","title":null,"venue":null,"work_id":"7ba59d12-413b-45f0-b74c-fb3828fd9ac9","year":2019},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":77,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.947627Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:bb48f86fd441dcadd179d854b791f2de7e35ab75c6fa1b313db5a9c372a07049","observation_id":"8aad0103-980a-4f90-b3b1-434d5e5ad1df","resolution":{"observed_at":"2026-08-12T11:30:28.813852Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.972494Z","title":null,"venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":78,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.972494Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:ea815d0b72046b8cab4a2aca47bdf43c4d095ac702282036c63491fb32588488","observation_id":"c3416b10-484a-4121-9ca1-2c6ecbb527e3","resolution":{"observed_at":"2026-08-12T11:30:27.972494Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.958729Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":79,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.958729Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:ade436116c9855da5709f19359c89b83c161e6925d5d9e54b224640a0be6a383","observation_id":"ce1cda8d-2c69-4e77-909c-ea84027c014b","resolution":{"observed_at":"2026-08-12T11:30:27.958729Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.981810Z","title":"Wiggins, Fahmid M","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":80,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.981810Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:a8a0e05f1ded7ae19a37e4a76830c458a7b1cdde09849fdcbc2a71b08da8e61a","observation_id":"c6bb094c-cda7-472d-81a9-b2148958f760","resolution":{"observed_at":"2026-08-12T11:30:27.981810Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.968087Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":81,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.968087Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:9a6d730382191b11b65e8dfb49326ab411fc9aac14ec0ca4fccab81eae742bd8","observation_id":"faf24bd6-7717-42fe-82c0-d9296e96e760","resolution":{"observed_at":"2026-08-12T11:30:27.968087Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.991617Z","title":"Bai, Robert Tairas, and Yu Huang","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":82,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.991617Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:2e9e2d8bc7db86af2add39d207acc4980e50997986c570baca8e7dc93e20c5f5","observation_id":"3a16b9ce-e337-46bb-816c-5ebd4432e74d","resolution":{"observed_at":"2026-08-12T11:30:27.991617Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.977093Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":83,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.977093Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:7e138f5ffdc4364ed64a528c91745fa0304476634d0f59c76cf19aa24665006d","observation_id":"f7a308c9-3677-4888-9622-3c91dc3d9b93","resolution":{"observed_at":"2026-08-12T11:30:27.977093Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.986744Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":85,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.986744Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:cd4454b594a41c262c9f55c166d1b8af0f9b4bad2121ecce218d524b409a71b9","observation_id":"17414038-65f2-4153-8c93-cd2830e2fb2b","resolution":{"observed_at":"2026-08-12T11:30:27.986744Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.995986Z","title":"Zamfirescu-Pereira, Richmond Y","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":87,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.995986Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:dd4347e845a09d160f49ddd7b356a726ad538221473111c809a9cfef4b875506","observation_id":"1a7018da-4c37-40a5-8d2f-8cd7fec552be","resolution":{"observed_at":"2026-08-12T11:30:27.995986Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.650787Z","title":"In Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems (Yokohama, Japan) (CHI ’21)","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.650787Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:e12a467f0bb27a46e619a42111387ab2daaae6d9b68c4b756e91e42d8528ada4","observation_id":"588635e6-980d-4115-bef7-b8bb5215a244","resolution":{"observed_at":"2026-08-12T11:30:27.650787Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:30:27.785836Z","title":"In Proceedings of the 2023 33 Brown et al","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","version":1},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-12T11:30:27.785836Z"},"links":{"citing_paper":"/paper/2411.18151"},"observation_digest":"sha256:4fa25ae9a7625e840bfc43f9eb0946384947a3319dacce8cb26a0d2cbd374529","observation_id":"4572e063-bf9f-4733-8dd0-f20aa1bc95f4","resolution":{"observed_at":"2026-08-12T11:30:27.785836Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2411.18151","last_updated":"2024-11-27T08:59:34Z","latest_version":1,"primary_category":"cs.CY","snapshot_observed_at":"2026-08-14T04:48:18.137520Z","submitted_at":"2024-11-27T08:59:34Z","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers"},"reference_resolution":{"displayed":80,"state_counts":{"malformed_identifier":3,"metadata_mismatch":11,"parse_uncertain":0,"unresolved":59,"verified_exact":2,"verified_fuzzy":5},"total_outbound_references":80},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"thesis":"As of 17 August 2026, this Paper Citation Record lists 80 of 80 outbound references and 0 inbound Pith citation observations for arXiv:2411.18151."}