from transformers import AutoTokenizerfrom vllm import LLM, SamplingParams model_dir = "."tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True)prompt = tokenizer.apply_chat_template( [{"role": "user", "content": "What is 2 + 3? Give a short answer."}], tokenize=False, add_generation_prompt=True,) llm = LLM( model=model_dir, tokenizer=model_dir, dtype="float16", max_model_len=4096, gpu_memory_utilization=0.75, max_num_seqs=1, limit_mm_per_prompt={"image": 0, "video": 0}, enforce_eager=True, enable_prefix_caching=False,)outputs = llm.generate([prompt], SamplingParams(temperature=0, max_tokens=256))print(outputs[0].outputs[0].text)