import torch
from transformers import AutoProcessor, AutoModelForMultimodalLM, AutoModelForCausalLM
MODEL_ID_HUB = "tepirale/gemma-4-12B-merge-coder40-agentic40-it20"
MA= "tepirale/gemma-4-12B-agentic-fable5-composer2.5-v2-3.5x-tau2-assistant-safetensors-yuxinlu1"
model = AutoModelForMultimodalLM.from_pretrained(
MODEL_ID_HUB,
dtype="auto",
device_map="auto",
# local_files_only=True
)
assistant_model = AutoModelForCausalLM.from_pretrained(MA,
dtype=torch.bfloat16,
device_map="auto"
)
processor = AutoProcessor.from_pretrained(MODEL_ID_HUB)
# Prompt - add image before text
messages = [
{
"role": "user", "content": [
{"type": "image", "url": "https://raw.githubusercontent.com/google-gemma/cookbook/refs/heads/main/apps/sample-data/GoldenGate.png"},
{"type": "text", "text": "What is shown in this image?"}
]
}
]
# Process input
inputs = processor.apply_chat_template(
messages,
tokenize=True,
return_dict=True,
return_tensors="pt",
add_generation_prompt=True,
enable_thinking=True
).to(model.device)
input_len = inputs["input_ids"].shape[-1]
# Generate output
outputs = model.generate(**inputs, max_new_tokens=3512, assistant_model=assistant_model)
response = processor.decode(outputs[0][input_len:], skip_special_tokens=False)
# Parse output
processor.parse_response(response)