c4_5pct_1e18_seed2_2026-07-05_00-24-50_063173-pt
Trained with nanochat. Checkpoint at step 3,368.
W&B run: https://wandb.ai/alexksternteam/optimal_seed_replicas_v0/runs/22xqm19i
Outcome
| metric | value |
|---|---|
step | 3368 |
smooth_train_loss | 3.2193498611450195 |
min_objective | 0.9792058169202794 |
flops_used | 9.498592991426642e+17 |
flops_per_token | 1075838976.0 |
total_training_time | 420.48487973213196 |
Training config
{
"model_pt": {
"sequence_len": 2048,
"vocab_size": 65536,
"n_layer": 12,
"n_head": 6,
"n_kv_head": 6,
"n_embd": 768
},
"model_ppt": {
"sequence_len": 2048,
"vocab_size": 65536,
"n_layer": 12,
"n_head": 6,
"n_kv_head": 6,
"n_embd": 768
},
"optim": {
"matrix_lr": 0.03,
"embedding_lr": 0.3,
"unembedding_lr": 0.004,
"weight_decay": 0.0
},
"train": {
"num_iterations": 1000,
"eval_every_n_steps": null,
"eval_every_flops_frac": null,
"save_every_n_steps": null,
"log_every_n_steps": 100,
"eval_at_end": true,
"save_at_end": true,
"grad_clip": 1.0,
"ema_beta": 0.0
},
"eval": {
"eval_steps": 8,
"eval_tokens": 2016000
},
"hardware": {
"device_batch_size": 32,
"grad_accum_steps": 1,
"peak_tflops": 2250.0
},
"wandb": {
"enabled": true,
"notes": "",
"group": "1e18_ppt0.05",
"tags": [
"nanochat_gpt",
"pt_fineweb-nanochatbpe-20B",
"ppt_c4-nanochatbpe-10B",
"seed_2",
"case_b",
"lr_trapezoid",
"depth_12"
],
"entity": null
},
"data_pt": "fineweb-nanochatbpe-20B",
"data_ppt": "c4-nanochatbpe-10B",
"extra_eval": [
"c4-nanochatbpe-10B"
],
"train_split": "train",
"pad_vocab": 65536,
"ppt_vocab_size": null,
"ppt_same_vocab_as_pt": true,
"ppt_unified_lr": true,
"lr_kind": "trapezoid",
"lr_warmup_ratio": 0.0,
"lr_warmdown_ratio": 0.4,
"lr_final_frac": 0.0,
"ppt_lr_kind": "trapezoid",
"ppt_lr_warmup_ratio": null,
"ppt_lr_warmdown_ratio": null,
"ppt_lr_final_frac": 0.0,
"ppt_lr": null,
"ppt_weight_decay": 0.0,
"eval_tokens": 2016000,
"reinit_embed_at_transition": false,
"reset_optimizer_at_transition": false,
"depth": 12,
"compile_model": true,
"model_project": "optimal_seed_replicas_v0",
"run_name": "c4_5pct_1e18_seed2",
"target_flops": 1e+18,
"alpha_ppt": 0.05,
"use_measured_flops": true,
"flops_per_token": null,
"ppt_flops_per_token": null,
"seed": 2,
"metrics_output_file": null,
"push_to_hf": true,
"hf_repo_org": "alexkstern",
"cleanup_checkpoints": true
}
Files
model_003368.pt— model weights (state_dict).meta_003368.json— training metadata.config_003368.json— run config snapshot.rng_003368.pt— RNG state (when present).