CiptakerLM v1

Dataset used for Fine-Tuning: Ciptaker-sft-data-preparation.ipynb Base model: sarahlintang/mistral-indo-7b Trained on 1x3090 @ 24 epochs

Train logs, metrics, and params: https://wandb.ai/willy030125/MistralCiptaker_v0.2_SFT/runs/c9so5vf8 Inference example using Colab T4: CiptakerLM-fine-tune-inference.ipynb Eval results using Colab T4: CiptakerLM-fine-tune-eval.ipynb

Prompt template:

### Human: {Instruction} ### Assistant: {response}

Usage example:

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoTokenizer, GenerationConfig

model_id = "Willy030125/CiptakerLM-v1"
device = "cuda" if torch.cuda.is_available() else "cpu"

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(model_id).to(device)

def create_instruction(instruction):
    prompt = f"### Human: {instruction} ### Assistant: "
    return prompt

def generate(
    instruction,
    max_new_tokens=2048,
    temperature=0.1,
    top_p=0.95,
    top_k=40,
    num_beams=4,
    **kwargs
):

    prompt = create_instruction(instruction)
    inputs = tokenizer(prompt, return_tensors="pt")
    input_ids = inputs["input_ids"].to(device)
    attention_mask = inputs["attention_mask"].to(device)
    generation_config = GenerationConfig(
        temperature=temperature,
        top_p=top_p,
        top_k=top_k,
        num_beams=num_beams,
        do_sample=True,
        **kwargs,
    )
    with torch.no_grad():
        generation_output = model.generate(
            input_ids=input_ids,
            attention_mask=attention_mask,
            generation_config=generation_config,
            pad_token_id=tokenizer.pad_token_id,
            eos_token_id=tokenizer.eos_token_id,
            return_dict_in_generate=True,
            output_scores=True,
            max_new_tokens=max_new_tokens,
            early_stopping=True
        )
    s = generation_output.sequences[0]
    output = tokenizer.decode(s, skip_special_tokens=True)
    return output.split("### Assistant:")[1].strip()

instruction = "Apa sanksi bagi pengusaha yang melanggar ketentuan dalam Pasal 42 ayat (2) tentang pekerja asing?"
print(generate(instruction))

Output:

Pengusaha dapat dikenai sanksi pidana penjara 1-4 tahun dan/atau denda antara Rp100.000.000 hingga Rp400.000.000.