import os
from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest
from transformers import AutoProcessor
from PIL import Image
MODEL_PATH = "Gyubeum/Qwen3-VL-8B-Instruct-UI-Genie"
BASE_MODEL = "Qwen/Qwen3-VL-8B-Instruct"
POS_TOKEN = "<|+|>"
NEG_TOKEN = "<|-|>"
JUDGE_PROMPT = (
"Was the agent's action above correct given the current screen state? "
"Answer with exactly one token: <|+|> for correct, <|-|> for wrong."
)
SYSTEM_PROMPT = (
"You are a helpful assistant.\n\n# Tools\n\n"
"You may call one or more functions to assist with the user query.\n\n"
"You are provided with function signatures within <tools></tools> XML tags:\n"
"<tools>\n"
'{"type": "function", "function": {"name": "mobile_use", '
'"description": "Use a touchscreen to interact with a mobile device. '
"The screen's resolution is 540x1200.\", "
'"parameters": {"properties": {"action": {"type": "string", '
'"enum": ["click", "long_press", "swipe", "type", "key", "system_button", "open", "wait", "terminate"]}, '
'"coordinate": {"type": "array"}, "text": {"type": "string"}, "button": {"type": "string"}}, '
'"required": ["action"]}}}\n</tools>\n\n'
"For each function call, return a json object within <tool_call></tool_call> XML tags:\n"
"<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call>"
)
llm = LLM(
model=BASE_MODEL,
dtype="bfloat16",
enable_lora=True,
max_lora_rank=64,
gpu_memory_utilization=0.7,
max_model_len=8192,
limit_mm_per_prompt={"image": 10},
enforce_eager=True,
)
processor = AutoProcessor.from_pretrained(BASE_MODEL, max_pixels=1_048_576)
tokenizer = processor.tokenizer
pos_id = tokenizer.encode(POS_TOKEN, add_special_tokens=False)[0]
neg_id = tokenizer.encode(NEG_TOKEN, add_special_tokens=False)[0]
lora_request = LoRARequest("rm_lora", 1, MODEL_PATH)
sampling_params = SamplingParams(max_tokens=1, temperature=0.0)
def score_action(goal, prior_steps_text, tool_call_response, screenshot):
"""
Score a GUI agent action.
Args:
goal: Task goal string.
prior_steps_text: String like "Step1: tap search\nStep2: type query\n"
tool_call_response: The agent's raw tool_call response string to judge.
screenshot: PIL.Image of the current screen state.
Returns:
float: 1.0 (correct), 0.0 (wrong), or 0.5 (undecided).
"""
user_content = [
{"type": "text", "text": f"The user query: {goal}\nTask progress (...): {prior_steps_text}; "},
{"type": "image"},
]
messages = [
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": user_content},
{"role": "assistant", "content": tool_call_response.strip()},
{"role": "user", "content": JUDGE_PROMPT},
]
prefix_text = processor.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
output = llm.generate(
[{"prompt": prefix_text, "multi_modal_data": {"image": [screenshot]}}],
sampling_params,
lora_request=lora_request,
)[0]
gen_id = output.outputs[0].token_ids[0] if output.outputs[0].token_ids else None
if gen_id == pos_id:
return 1.0
if gen_id == neg_id:
return 0.0
return 0.5
screenshot = Image.open("screenshot.png").convert("RGB")
tool_call = '<tool_call>\n{"name": "mobile_use", "arguments": {"action": "click", "coordinate": [540, 120]}}\n</tool_call>'
score = score_action(
goal="Tap the search button",
prior_steps_text="Step1: opened the app\n",
tool_call_response=tool_call,
screenshot=screenshot,
)
print(f"Score: {score}")
