import re from PIL import Imagefrom transformers import AutoModelForImageTextToText, AutoProcessor repo = "shricodev/chashmish-babubhaiya"model = AutoModelForImageTextToText.from_pretrained(repo, dtype="auto", device_map="auto")processor = AutoProcessor.from_pretrained(repo) image = Image.open("screenshot.png").convert("RGB")instruction = "open the pricing page"prompt = ( f"Point to the UI element I should click to: {instruction}. " "Answer with the click point as (x, y).")content = [{"type": "image"}, {"type": "text", "text": prompt}]messages = [{"role": "user", "content": content}]text = processor.apply_chat_template(messages, add_generation_prompt=True)inputs = processor(text=text, images=[image], return_tensors="pt").to(model.device) out = model.generate(**inputs, max_new_tokens=16, do_sample=False)reply = processor.decode(out[0, inputs["input_ids"].shape[1]:], skip_special_tokens=True)x, y = (int(v) for v in re.findall(r"\d+", reply)[:2])print(x * image.width / 1000, y * image.height / 1000) # click point in pixels