{"schema_version":"1.0","language":"zh-CN","generated_from":"NEWVAR 从零开始学AI structured Markdown","write_access":false,"full_text_access":false,"content_boundary":"This endpoint exposes metadata, objectives, prerequisites, labs and source summaries. It does not expose chapter body text or answer text.","chapter":{"slug":"pretraining-data-scaling","title":"预训练目标、数据与缩放","english_title":"Pretraining, data and scaling","chapter_number":79,"volume":4,"volume_title":"第四卷：前沿、系统与研究","month":20,"unit_title":"基础模型和语言模型从零实现","review_status":"draft","updated_at":"2026-08-06","estimated_hours":10,"focus":"理解下一个词预测、数据混合、去重、训练计算和缩放规律的适用边界。","objectives":["理解下一个词预测、数据混合、去重、训练计算和缩放规律的适用边界。","解释核心数学关系并完成最小实现","运行单变量实验并记录失败样例","完成工程线或研究线至少一项迁移任务"],"prerequisites":["transformer-from-scratch"],"exercise_count":3,"tracks":["engineering","research"],"human_url":"/learn/pretraining-data-scaling"},"labs":[{"id":"lab-pretraining-data-scaling","chapter_slug":"pretraining-data-scaling","title":"预训练目标、数据与缩放：单变量实验","kind":"data_tokens","data_status":"教学模拟","variable":{"label":"训练词元","unit":"token","min":1000,"max":1000000,"step":1000,"default":100000},"objective":"只改变“训练词元”，观察结果、代价和风险如何一起变化，理解理解下一个词预测、数据混合、去重、训练计算和缩放规律的适用边界。","static_fallback":"静态替代：把训练词元分别设为1000token、100000token和1000000token，手工比较三组教学模拟输出。","keyboard":"聚焦滑块后使用方向键微调，Page Up/Page Down大步调整，Home/End到达边界。","measurement_note":"页面数值由公开公式生成，只用于教学，不代表真实模型性能或现实世界因果效果。","source_ids":["S030","S050","S051","S052","S053","S054","S055"]}],"project":{"id":"project-20","month":20,"title":"从零训练一个微型语言模型","brief":"在CPU小语料上实现词元器、因果注意力、训练、采样和评估；记录数据许可与算力边界。","deliverables":["问题与边界说明","可运行最小实现","实验记录与失败分析","来源与许可清单","风险、隐私与人工闸门","复现README"]},"sources":[{"id":"S030","author":"Ashish Vaswani et al.","title":"Attention Is All You Need","source_level":"同行评审论文预印本","published_at":"2017","accessed_at":"2026-08-06","doi_or_url":"https://arxiv.org/abs/1706.03762"},{"id":"S050","author":"Stanford University","title":"CS336: Language Modeling from Scratch","source_level":"大学课程一手资料","published_at":"2026","accessed_at":"2026-08-06","doi_or_url":"https://cs336.stanford.edu/"},{"id":"S051","author":"Taku Kudo, John Richardson","title":"SentencePiece","source_level":"同行评审论文预印本","published_at":"2018","accessed_at":"2026-08-06","doi_or_url":"https://arxiv.org/abs/1808.06226"},{"id":"S052","author":"Tom Brown et al.","title":"Language Models are Few-Shot Learners","source_level":"同行评审论文预印本","published_at":"2020","accessed_at":"2026-08-06","doi_or_url":"https://arxiv.org/abs/2005.14165"},{"id":"S053","author":"Jordan Hoffmann et al.","title":"Training Compute-Optimal Large Language Models","source_level":"同行评审论文预印本","published_at":"2022","accessed_at":"2026-08-06","doi_or_url":"https://arxiv.org/abs/2203.15556"},{"id":"S054","author":"Long Ouyang et al.","title":"Training language models to follow instructions with human feedback","source_level":"同行评审论文预印本","published_at":"2022","accessed_at":"2026-08-06","doi_or_url":"https://arxiv.org/abs/2203.02155"},{"id":"S055","author":"Patrick Lewis et al.","title":"Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks","source_level":"同行评审论文预印本","published_at":"2020","accessed_at":"2026-08-06","doi_or_url":"https://arxiv.org/abs/2005.11401"}]}