code_5pct_3e18_seed2_2026-07-12_02-10-08_098372-ppt
Trained with nanochat. Checkpoint at step 275.
W&B run: https://wandb.ai/alexksternteam/optimal_seed_replicas_v1/runs/icxikmft
Outcome
| metric | value |
|---|---|
step | 275 |
smooth_train_loss | 2.775472640991211 |
min_objective | 1.7841455652069105 |
flops_used | 1.499733860286464e+17 |
flops_per_token | 2080374784.0 |
total_training_time | 51.19044303894043 |
Training config
{
"model_pt": {
"sequence_len": 2048,
"vocab_size": 65536,
"n_layer": 16,
"n_head": 8,
"n_kv_head": 8,
"n_embd": 1024
},
"model_ppt": {
"sequence_len": 2048,
"vocab_size": 65536,
"n_layer": 16,
"n_head": 8,
"n_kv_head": 8,
"n_embd": 1024
},
"optim": {
"matrix_lr": 0.015,
"embedding_lr": 0.3,
"unembedding_lr": 0.004,
"weight_decay": 0.0
},
"train": {
"num_iterations": 1000,
"eval_every_n_steps": null,
"eval_every_flops_frac": null,
"save_every_n_steps": null,
"log_every_n_steps": 100,
"eval_at_end": true,
"save_at_end": true,
"grad_clip": 1.0,
"ema_beta": 0.0
},
"eval": {
"eval_steps": 8,
"eval_tokens": 2016000
},
"hardware": {
"device_batch_size": 32,
"grad_accum_steps": 1,
"peak_tflops": 2250.0
},
"wandb": {
"enabled": true,
"notes": "",
"group": "3e18_ppt0.05",
"tags": [
"nanochat_gpt",
"pt_fineweb-nanochatbpe-20B",
"ppt_github-code-nanochatbpe-1B",
"seed_2",
"case_b",
"lr_trapezoid",
"depth_16"
],
"entity": null
},
"data_pt": "fineweb-nanochatbpe-20B",
"data_ppt": "github-code-nanochatbpe-1B",
"extra_eval": [
"c4-nanochatbpe-10B"
],
"train_split": "train",
"pad_vocab": 65536,
"ppt_vocab_size": null,
"ppt_same_vocab_as_pt": true,
"ppt_unified_lr": true,
"lr_kind": "trapezoid",
"lr_warmup_ratio": 0.02,
"lr_warmdown_ratio": 0.5,
"lr_final_frac": 0.0,
"ppt_lr_kind": "trapezoid",
"ppt_lr_warmup_ratio": null,
"ppt_lr_warmdown_ratio": null,
"ppt_lr_final_frac": 0.0,
"ppt_lr": null,
"ppt_weight_decay": 0.0,
"ppt_device_batch_size": null,
"ppt_grad_accum_steps": null,
"eval_tokens": 2016000,
"reinit_embed_at_transition": false,
"reset_optimizer_at_transition": false,
"depth": 16,
"compile_model": true,
"model_project": "optimal_seed_replicas_v1",
"run_name": "code_5pct_3e18_seed2",
"target_flops": 3e+18,
"alpha_ppt": 0.05,
"use_measured_flops": true,
"flops_per_token": null,
"ppt_flops_per_token": null,
"seed": 2,
"metrics_output_file": null,
"push_to_hf": true,
"hf_repo_org": "alexkstern",
"cleanup_checkpoints": true
}
Files
model_000275.pt— model weights (state_dict).meta_000275.json— training metadata.config_000275.json— run config snapshot.rng_000275.pt— RNG state (when present).