This is 82M parameters llama model of random weights. This model can be use for proof of concept. Tokenizer is copy of meta-llama/Llama-2-7b
# Use a pipeline as a high-level helper
from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer
import numpy as np
config = LlamaConfig(vocab_size=32000, hidden_size=768, intermediate_size=768*4, num_hidden_layers=4, num_attention_heads=8)
tokenizer = LlamaTokenizer.from_pretrained("meta-llama/Llama-2-7b")
model = LlamaForCausalLM(config).half()
model_parameters = filter(lambda p: p.requires_grad, model.parameters())
params = sum([np.prod(p.size()) for p in model_parameters])
print(params / 1024 / 1024) # 82.881591796875
hub_id = "heegyu/llama-small-randomweights"
tokenizer.push_to_hub(hub_id)
model.push_to_hub(hub_id)