Dzongkha GPT-2 Large (Next-Word Prediction)
This repository contains the weights for the GPT-2 Large variant trained specifically for the Dzongkha language. It is designed for causal language modeling and interactive next-word prediction tasks.
📐 Model Architecture & Parameters
- Architecture: GPT-2 Large
- Layers (N_L): 36
- Attention Heads (N_H): 20
- Embedding Dimension (d_model): 1280
- Parameters: ~774M
- Tokenizer: Custom Fast Tokenizer (
PreTrainedTokenizerFast)
🚀 Quickstart & Inference
import torch import re from transformers import GPT2LMHeadModel, PreTrainedTokenizerFast
1. LOAD MODEL & TOKENIZER ---
REPO_ID = "KarmaCST/dzongkha_nextword_gpt2" device = "cuda" if torch.cuda.is_available() else "cpu"
print("Loading model from Hugging Face...") tokenizer = PreTrainedTokenizerFast.from_pretrained(REPO_ID) model = GPT2LMHeadModel.from_pretrained(REPO_ID).to(device) model.eval()
2. PREDICTION FUNCTION ---
def predict_next_5_words(text): input_ids = tokenizer.encode(text, return_tensors="pt").to(device)
with torch.no_grad():
logits = model(input_ids).logits[:, -1, :]
probs = torch.nn.functional.softmax(logits, dim=-1)
# Get top probability candidates
top_probs, top_indices = torch.topk(probs, 50)
predictions = []
for idx, prob in zip(top_indices[0], top_probs[0]):
word = tokenizer.decode([idx]).strip()
# Filter out empty strings or byte-fallback artifacts like <0x..>
if len(word) > 0 and not re.search(r'<0x[0-9A-Fa-f]+>', word):
predictions.append((word, prob.item()))
if len(predictions) == 5:
break
return predictions
3. RUN & PRINT ---
input_word = "འབྲུག་རྒྱལ་ཁབ་"
print(f"\nGiven Word: {input_word}") print("=" * 35)
top_5 = predict_next_5_words(input_word)
for rank, (word, prob) in enumerate(top_5, 1): print(f"{rank}. {word:<15} (Probability: {prob:.2%})")