{ "product": "nanochat (Python)", "title": "nanochat (Python)", "summary": "Karpathy's nanochat — tokenizer, pretraining, fine-tuning, RL and inference.", "category": "ports", "source": "D:\\Dev\\Lab\\NanoChat.Upstream", "entries": [ { "path": [ "AI", "Tokenizer", "BPE tokenizer training" ], "value": "rustbpe", "status": "yes", "note": "nanochat/tokenizer.py, scripts/tok_train.py" }, { "path": [ "AI", "Tokenizer", "GPT-4-style split pattern" ], "value": "", "status": "yes", "note": "regex in nanochat/tokenizer.py, \\p{N}{1,2} variant" }, { "path": [ "AI", "Tokenizer", "tiktoken interop" ], "value": "", "status": "yes", "note": "merges -> tiktoken.Encoding for fast encode/decode" }, { "path": [ "AI", "Tokenizer", "Chat special tokens" ], "value": "9 specials", "status": "yes", "note": "bos, user/assistant, python/output start-end" }, { "path": [ "AI", "Tokenizer", "Conversation render + loss mask" ], "value": "", "status": "yes", "note": "render_conversation: assistant supervised, tool output not" }, { "path": [ "AI", "Tokenizer", "Tokenizer persistence" ], "value": "pickle", "status": "yes", "note": "~/.cache/nanochat/tokenizer round-trip" }, { "path": [ "AI", "Tokenizer", "Compression metrics" ], "value": "", "status": "yes", "note": "scripts/tok_eval.py vs GPT-2/GPT-4 tokenizers" }, { "path": [ "AI", "Data pipeline", "ClimbMix shard downloader" ], "value": "", "status": "yes", "note": "nanochat/dataset.py: parallel, retries, atomic rename" }, { "path": [ "AI", "Data pipeline", "In-app Parquet ingestion" ], "value": "", "status": "yes", "note": "row-group streaming, rank-strided" }, { "path": [ "AI", "Data pipeline", "BOS best-fit packing loader" ], "value": "", "status": "yes", "note": "nanochat/dataloader.py, ~100% row utilization" }, { "path": [ "AI", "Data pipeline", "Resumable loader state" ], "value": "", "status": "yes", "note": "{pq_idx, rg_idx, epoch} resume dict" }, { "path": [ "AI", "Data pipeline", "SFT conversation packing" ], "value": "", "status": "yes", "note": "chat_sft.py best-fit packing, pad-never-crop" }, { "path": [ "AI", "Data pipeline", "Upstream RNG reproduction" ], "value": "", "status": "na", "note": "is the reference implementation" }, { "path": [ "AI", "Model & kernels", "GPT transformer (RoPE, QK-norm, relu2)" ], "value": "", "status": "yes", "note": "nanochat/gpt.py, logit softcap 15, untied embeddings" }, { "path": [ "AI", "Model & kernels", "Grouped-query attention" ], "value": "", "status": "yes", "note": "separate n_kv_head" }, { "path": [ "AI", "Model & kernels", "Sliding-window attention" ], "value": "SSSL", "status": "yes", "note": "window_pattern tiled per layer" }, { "path": [ "AI", "Model & kernels", "Value embeddings + gates" ], "value": "", "status": "yes", "note": "ResFormer-style, sigmoid gate" }, { "path": [ "AI", "Model & kernels", "Smear + backout" ], "value": "", "status": "yes", "note": "prev-token gate; mid-layer residual backout" }, { "path": [ "AI", "Model & kernels", "Optimized attention kernels" ], "value": "FA3", "status": "yes", "note": "Flash Attention 3 with SDPA fallback (flash_attention.py)" }, { "path": [ "AI", "Model & kernels", "Backward pass" ], "value": "autograd", "status": "yes", "note": "PyTorch autograd" }, { "path": [ "AI", "Pretraining", "Base training loop" ], "value": "", "status": "yes", "note": "scripts/base_train.py: grad-accum, compile, MFU/ETA" }, { "path": [ "AI", "Pretraining", "Muon + AdamW optimizer" ], "value": "", "status": "yes", "note": "Polar Express, NorMuon, MuonEq, cautious decay (optim.py)" }, { "path": [ "AI", "Pretraining", "LR/momentum/WD schedules" ], "value": "", "status": "yes", "note": "warmup-constant-warmdown; momentum 0.85-0.97-0.90" }, { "path": [ "AI", "Pretraining", "Scaling-law autotuning" ], "value": "", "status": "yes", "note": "horizon from param-data ratio; B ~ D^0.383" }, { "path": [ "AI", "Pretraining", "Gradient accumulation" ], "value": "", "status": "yes", "note": "base_train.py" }, { "path": [ "AI", "Pretraining", "Resume mid-run" ], "value": "", "status": "yes", "note": "model + per-rank optimizer shard + loader position" }, { "path": [ "AI", "Pretraining", "In-training evals & samples" ], "value": "", "status": "yes", "note": "val bpb, CORE, greedy samples, wandb" }, { "path": [ "AI", "SFT", "SFT training flow" ], "value": "", "status": "yes", "note": "scripts/chat_sft.py" }, { "path": [ "AI", "SFT", "Data mixture (SmolTalk+MMLU+GSM8K)" ], "value": "", "status": "yes", "note": "460K + aux x3 + GSM8K x4, deterministic shuffle" }, { "path": [ "AI", "SFT", "Progress-driven LR schedule" ], "value": "", "status": "yes", "note": "keyed to dataset progress 0-1" }, { "path": [ "AI", "SFT", "Optimizer warm-start from base" ], "value": "", "status": "yes", "note": "loads pretrain momentum buffers per rank" }, { "path": [ "AI", "Reinforcement learning", "GRPO rollouts & advantages" ], "value": "", "status": "yes", "note": "chat_rl.py: mean-subtracted advantage, DAPO-style norm" }, { "path": [ "AI", "Reinforcement learning", "RL training loop" ], "value": "", "status": "yes", "note": "on-policy REINFORCE on GSM8K, periodic checkpoints" }, { "path": [ "AI", "Reinforcement learning", "pass@k evaluation" ], "value": "", "status": "yes", "note": "all-reduced pass@1..k during RL" }, { "path": [ "AI", "Evaluation", "CORE benchmark" ], "value": "22 tasks", "status": "yes", "note": "core_eval.py: MC, schema, LM scoring" }, { "path": [ "AI", "Evaluation", "Bits-per-byte metric" ], "value": "", "status": "yes", "note": "loss_eval.py, vocab-size invariant" }, { "path": [ "AI", "Evaluation", "ChatCORE aggregate" ], "value": "", "status": "yes", "note": "ARC-E/C, MMLU, GSM8K, HumanEval, baseline-centered" }, { "path": [ "AI", "Evaluation", "GSM8K evaluation" ], "value": "", "status": "yes", "note": "tasks/gsm8k.py, tool-call parsing" }, { "path": [ "AI", "Evaluation", "HumanEval execution" ], "value": "", "status": "yes", "note": "sandboxed subprocess: rlimit, scrubbed env (execution.py)" }, { "path": [ "AI", "Evaluation", "MMLU/ARC categorical eval" ], "value": "", "status": "yes", "note": "letter-logit restriction (chat_eval.py)" }, { "path": [ "AI", "Evaluation", "Inference benchmark" ], "value": "", "status": "yes", "note": "infer_bench.py: TTFT/TPOT/MBU/MFU sweep" }, { "path": [ "AI", "Evaluation", "Durable eval history & integrity" ], "value": "", "status": "no", "note": "CSV results only; no integrity gates or run history" }, { "path": [ "AI", "Inference engine", "KV-cache incremental decode" ], "value": "", "status": "yes", "note": "engine.py KVCache, FA3 layout" }, { "path": [ "AI", "Inference engine", "Prefill-once, clone-for-samples" ], "value": "", "status": "yes", "note": "batch-1 prefill copied into n-row decode cache" }, { "path": [ "AI", "Inference engine", "Streaming generation" ], "value": "", "status": "yes", "note": "token-column streaming with sampled/forced masks" }, { "path": [ "AI", "Inference engine", "Temperature/top-k sampling" ], "value": "", "status": "yes", "note": "seeded generator, 0=argmax" }, { "path": [ "AI", "Inference engine", "Calculator tool-use loop" ], "value": "", "status": "yes", "note": "python_start/end state machine + safe eval" }, { "path": [ "AI", "Inference engine", "Multi-turn chat session" ], "value": "", "status": "yes", "note": "chat_cli.py token accumulation" }, { "path": [ "AI", "Inference engine", "CPU fallback inference" ], "value": "", "status": "yes", "note": "device autodetect cuda/mps/cpu" }, { "path": [ "UI", "Chat CLI" ], "value": "", "status": "yes", "note": "scripts/chat_cli.py REPL + one-shot mode" }, { "path": [ "UI", "Web chat UI" ], "value": "", "status": "no", "note": "ui/ folder empty in this revision" }, { "path": [ "UI", "MCP server" ], "value": "", "status": "no", "note": "" }, { "path": [ "UI", "Desktop observatory app" ], "value": "", "status": "no", "note": "" }, { "path": [ "UI", "Architecture visualizers" ], "value": "", "status": "no", "note": "" }, { "path": [ "UI", "Interactive tokenizer explorer" ], "value": "", "status": "no", "note": "visualize_tokenization ANSI dump only" }, { "path": [ "UI", "Training workflow UI" ], "value": "", "status": "no", "note": "shell scripts (runs/speedrun.sh)" }, { "path": [ "UI", "End-to-end speedrun script" ], "value": "", "status": "yes", "note": "runs/speedrun.sh: 8xH100 ~1.5h pipeline" }, { "path": [ "AI", "Infrastructure", "Checkpoint save/load + resume" ], "value": "", "status": "yes", "note": "checkpoint_manager.py, ZeRO-2 optimizer shards" }, { "path": [ "AI", "Infrastructure", "Model lineage registry (base/sft/rl)" ], "value": "", "status": "yes", "note": "load_model source registry" }, { "path": [ "AI", "Infrastructure", "Model deploy/rollback registry" ], "value": "", "status": "no", "note": "" }, { "path": [ "AI", "Infrastructure", "Artifact integrity (hashes)" ], "value": "", "status": "no", "note": "" }, { "path": [ "AI", "Infrastructure", "Cross-implementation checkpoint import" ], "value": "", "status": "na", "note": "is the source of checkpoints" }, { "path": [ "AI", "Infrastructure", "Distributed multi-GPU training" ], "value": "", "status": "yes", "note": "custom reduce_scatter/all_gather in optimizer, no DDP" }, { "path": [ "AI", "Infrastructure", "Mixed precision (bf16/FP8)" ], "value": "", "status": "yes", "note": "bf16 autodetect + ~150-line FP8 path (fp8.py)" }, { "path": [ "AI", "Infrastructure", "Config provenance tracking" ], "value": "", "status": "no", "note": "flags logged but not chosen-vs-derived tracked" } ] }