from transformers import AutoProcessor, Qwen2VLForConditionalGeneration
from peft import PeftModel
from PIL import Image
from huggingface_hub import snapshot_download
adapter_dir = snapshot_download(repo_id='devanshty/Babel')
base_model = Qwen2VLForConditionalGeneration.from_pretrained(
"Qwen/Qwen2-VL-7B-Instruct",
torch_dtype="auto",
device_map="auto"
)
processor = AutoProcessor.from_pretrained(adapter_dir)
model = PeftModel.from_pretrained(base_model, adapter_dir)
model.eval()
image = Image.open("document.jpg")
messages = [
{
"role": "user",
"content": [
{"type": "image", "image": image},
{"type": "text", "text": "Extract all text from this image and translate it to English."}
]
}
]
text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = processor(text=[text], images=[image], return_tensors="pt").to(model.device)
output = model.generate(**inputs, max_new_tokens=1024)
print(processor.decode(output[0], skip_special_tokens=True))