• Foundation Model Bllossom 8B
• datasets
  • jojo0217/korean_safe_conversation

Usage for Transformers

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline

BASE_MODEL = "sh2orc/Llama-3-Korean-8B"

model = AutoModelForCausalLM.from_pretrained(BASE_MODEL,
    torch_dtype=torch.bfloat16,
    attn_implementation="flash_attention_2", 
    device_map="cuda:0")

tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = 'right'

instruction = '할부 결제 대해서 설명해줘'

pipe = pipeline("text-generation", 
                model=model, 
                tokenizer=tokenizer, 
                max_new_tokens=1024)

messages = [
    {"role": "user", "content": instruction},
]

prompt = pipe.tokenizer.apply_chat_template(
        messages, 
        tokenize=False, 
        add_generation_prompt=True
)

outputs = pipe(
    prompt,
    do_sample=True,
    temperature=0.8,
    top_k=10,
    top_p=0.9,
    add_special_tokens=True,
    eos_token_id = [ 
        pipe.tokenizer.eos_token_id,
        pipe.tokenizer.convert_tokens_to_ids("<|eot_id|>")
    ]
)

print(outputs[0]['generated_text'][len(prompt):])
print(outputs[0]['generated_text'][len(prompt):])

Result

Usage for VLLM

from vllm import LLM, SamplingParams
from transformers import AutoTokenizer, pipeline

BASE_MODEL = "sh2orc/Llama-3-Korean-8B"

llm = LLM(model=BASE_MODEL)

tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = 'right'

instruction = '카드 할부 결제에 대해서 알려줘'

messages = [
    {
      "role": "system",
      "content": "당신은 훌륭한 AI 비서입니다. You are a great AI assistant."
    },
    {
      "role": "user",
      "content": instruction
    }, 
]


prompt_message = tokenizer.apply_chat_template(
        messages, 
        tokenize=False, 
        add_generation_prompt=True,
)

eos_token_id = [tokenizer.eos_token_id, tokenizer.convert_tokens_to_ids("<|eot_id|>")]

outputs = llm.generate(prompt_message, SamplingParams(stop_token_ids=eos_token_id, temperature=0.6, top_p=0.8,max_tokens=4096))

for output in outputs:
    propt = output.prompt
    generated_text = output.outputs[0].text
    print(generated_text)

Result