{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:EWI4AYZHQYNRDBOLDY6CVQIRPL","short_pith_number":"pith:EWI4AYZH","schema_version":"1.0","canonical_sha256":"2591c06327861b1185cb1e3c2ac1117adb94ec48a5621d1c95fa6646114c44fd","source":{"kind":"arxiv","id":"2311.08993","version":1},"attestation_state":"computed","paper":{"title":"When does In-context Learning Fall Short and Why? A Study on Specification-Heavy Tasks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Bin Xu, Hao Peng, Jianhui Chen, Juanzi Li, Kaisheng Zeng, Lei Hou, Weikai Li, Xiaozhi Wang, Yunjia Qi, Zhili Wu, Zimu Wang","submitted_at":"2023-11-15T14:26:30Z","abstract_excerpt":"In-context learning (ICL) has become the default method for using large language models (LLMs), making the exploration of its limitations and understanding the underlying causes crucial. In this paper, we find that ICL falls short of handling specification-heavy tasks, which are tasks with complicated and extensive task specifications, requiring several hours for ordinary humans to master, such as traditional information extraction tasks. The performance of ICL on these tasks mostly cannot reach half of the state-of-the-art results. To explore the reasons behind this failure, we conduct compre"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2311.08993","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-15T14:26:30Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"afa6e833caea7152812cb78c63ac81f2623451bceaaa762d9e6e866035c90512","abstract_canon_sha256":"f5dbb0943e309d006ebd540a5bc9b23f4866e7266c6c1d3bf4d7aa6f9d788fbf"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:13:07.473239Z","signature_b64":"j1lF8yZW9qPOElW+B00e8HdcX5SkFgISGq/Y6Tna4h4wPeZgTN441d6B5rZu05z1MYlzcE21gwF+FnAisiHABA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2591c06327861b1185cb1e3c2ac1117adb94ec48a5621d1c95fa6646114c44fd","last_reissued_at":"2026-07-05T07:13:07.472769Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:13:07.472769Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"When does In-context Learning Fall Short and Why? A Study on Specification-Heavy Tasks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Bin Xu, Hao Peng, Jianhui Chen, Juanzi Li, Kaisheng Zeng, Lei Hou, Weikai Li, Xiaozhi Wang, Yunjia Qi, Zhili Wu, Zimu Wang","submitted_at":"2023-11-15T14:26:30Z","abstract_excerpt":"In-context learning (ICL) has become the default method for using large language models (LLMs), making the exploration of its limitations and understanding the underlying causes crucial. In this paper, we find that ICL falls short of handling specification-heavy tasks, which are tasks with complicated and extensive task specifications, requiring several hours for ordinary humans to master, such as traditional information extraction tasks. The performance of ICL on these tasks mostly cannot reach half of the state-of-the-art results. To explore the reasons behind this failure, we conduct compre"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.08993","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2311.08993/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2311.08993","created_at":"2026-07-05T07:13:07.472828+00:00"},{"alias_kind":"arxiv_version","alias_value":"2311.08993v1","created_at":"2026-07-05T07:13:07.472828+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.08993","created_at":"2026-07-05T07:13:07.472828+00:00"},{"alias_kind":"pith_short_12","alias_value":"EWI4AYZHQYNR","created_at":"2026-07-05T07:13:07.472828+00:00"},{"alias_kind":"pith_short_16","alias_value":"EWI4AYZHQYNRDBOL","created_at":"2026-07-05T07:13:07.472828+00:00"},{"alias_kind":"pith_short_8","alias_value":"EWI4AYZH","created_at":"2026-07-05T07:13:07.472828+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2505.06120","citing_title":"LLMs Get Lost In Multi-Turn Conversation","ref_index":62,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12185","citing_title":"Mitigating Context-Memory Conflicts in LLMs through Dynamic Cognitive Reconciliation Decoding","ref_index":32,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/EWI4AYZHQYNRDBOLDY6CVQIRPL","json":"https://pith.science/pith/EWI4AYZHQYNRDBOLDY6CVQIRPL.json","graph_json":"https://pith.science/api/pith-number/EWI4AYZHQYNRDBOLDY6CVQIRPL/graph.json","events_json":"https://pith.science/api/pith-number/EWI4AYZHQYNRDBOLDY6CVQIRPL/events.json","paper":"https://pith.science/paper/EWI4AYZH"},"agent_actions":{"view_html":"https://pith.science/pith/EWI4AYZHQYNRDBOLDY6CVQIRPL","download_json":"https://pith.science/pith/EWI4AYZHQYNRDBOLDY6CVQIRPL.json","view_paper":"https://pith.science/paper/EWI4AYZH","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2311.08993&json=true","fetch_graph":"https://pith.science/api/pith-number/EWI4AYZHQYNRDBOLDY6CVQIRPL/graph.json","fetch_events":"https://pith.science/api/pith-number/EWI4AYZHQYNRDBOLDY6CVQIRPL/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/EWI4AYZHQYNRDBOLDY6CVQIRPL/action/timestamp_anchor","attest_storage":"https://pith.science/pith/EWI4AYZHQYNRDBOLDY6CVQIRPL/action/storage_attestation","attest_author":"https://pith.science/pith/EWI4AYZHQYNRDBOLDY6CVQIRPL/action/author_attestation","sign_citation":"https://pith.science/pith/EWI4AYZHQYNRDBOLDY6CVQIRPL/action/citation_signature","submit_replication":"https://pith.science/pith/EWI4AYZHQYNRDBOLDY6CVQIRPL/action/replication_record"}},"created_at":"2026-07-05T07:13:07.472828+00:00","updated_at":"2026-07-05T07:13:07.472828+00:00"}