- model SafeTensors
- model convert GGUF to SafeTensors
https://huggingface.co/yuxinlu1/gemma-4-12B-agentic-fable5-composer2.5-v2-3.5x-tau2-GGUF
- model 'gemma4-v2-Q8_0.gguf'
- model convert by
https://huggingface.co/NickyNicky
import torch
from transformers import AutoProcessor, AutoModelForImageTextToText, AutoModelForCausalLM
M = "tepirale/gemma-4-12B-agentic-fable5-composer2.5-v2-3.5x-tau2-safetensors-yuxinlu1" # la misma carpeta, ahora con embedders
MA= "tepirale/gemma-4-12B-agentic-fable5-composer2.5-v2-3.5x-tau2-assistant-safetensors-yuxinlu1"
model = AutoModelForImageTextToText.from_pretrained(M, dtype=torch.bfloat16, device_map="auto")
assistant_model = AutoModelForCausalLM.from_pretrained(MA, dtype=torch.bfloat16, device_map="auto")
processor = AutoProcessor.from_pretrained(M)
# Prompt - add image before text
messages = [
{
"role": "user", "content": [
{"type": "image", "url": "https://raw.githubusercontent.com/google-gemma/cookbook/refs/heads/main/apps/sample-data/GoldenGate.png"},
{"type": "text", "text": "What is shown in this image?"}
]
}
]
# Process input
inputs = processor.apply_chat_template(
messages,
tokenize=True,
return_dict=True,
return_tensors="pt",
add_generation_prompt=True,
enable_thinking=False, # False True
).to(model.device)
input_len = inputs["input_ids"].shape[-1]
# Generate output
outputs = model.generate(**inputs, max_new_tokens=512, assistant_model=assistant_model)
response = processor.decode(outputs[0][input_len:], skip_special_tokens=False)
# Parse output
processor.parse_response(response)
''' response
{'content': 'The Golden Gate Bridge, painted in its iconic International Orange, spans San Francisco Bay in California. In the foreground, a lone seagull perches atop a rocky outcrop rising from the water. The Alcatraz Island penitentiary is visible below the bridge.',
'role': 'assistant'}
'''