import jsonimport osfrom collections.abc import Mappingfrom pathlib import Path # 在导入推理库之前启用离线模式,不下载任何模型文件。os.environ["HF_HUB_OFFLINE"] = "1"os.environ["TRANSFORMERS_OFFLINE"] = "1" MODEL_DIR = Path(".").resolve() # 从下载后的完整模型目录运行 def token_ids_from_template(encoded): # Transformers 5 可能返回 BatchEncoding/UserDict,而不一定是普通 dict。 if isinstance(encoded, Mapping): if "input_ids" not in encoded: raise ValueError("Chat template 未返回 input_ids") encoded = encoded["input_ids"] if hasattr(encoded, "tolist"): encoded = encoded.tolist() if (isinstance(encoded, (list, tuple)) and len(encoded) == 1 and isinstance(encoded[0], (list, tuple))): encoded = encoded[0] if not isinstance(encoded, (list, tuple)) or not encoded: raise TypeError("Chat template 必须返回非空 token ID 序列") if any(isinstance(token, bool) or not isinstance(token, int) or token < 0 for token in encoded): raise TypeError("Token ID 必须为非负整数") return list(encoded) def main(): from vllm import LLM, SamplingParams if not MODEL_DIR.is_dir() or not (MODEL_DIR / "config.json").is_file(): raise FileNotFoundError("请将 MODEL_DIR 改为完整的本地模型目录") if not list(MODEL_DIR.glob("*.safetensors")): raise FileNotFoundError("模型权重缺失,不能使用本地占位目录") config = json.loads((MODEL_DIR / "config.json").read_text(encoding="utf-8")) if config.get("quantization_config", {}).get("quant_method") != "compressed-tensors": raise ValueError("该示例要求本模型的 compressed-tensors checkpoint") generation_path = MODEL_DIR / "generation_config.json" generation = json.loads(generation_path.read_text(encoding="utf-8")) if generation_path.exists() else {} eos = generation.get("eos_token_id") if eos is None: eos = config.get("eos_token_id") if eos is None: eos = config.get("text_config", {}).get("eos_token_id") stop_ids = eos if isinstance(eos, list) else ([eos] if eos is not None else []) llm = LLM( model=str(MODEL_DIR), tokenizer=str(MODEL_DIR), dtype="float16", tensor_parallel_size=1, max_model_len=4096, gpu_memory_utilization=0.75, enforce_eager=True, max_num_seqs=1, enable_prefix_caching=False, limit_mm_per_prompt={"image": 0, "video": 0}, generation_config="vllm", trust_remote_code=False, seed=42, ) tokenizer = llm.get_tokenizer() messages = [{"role": "user", "content": "小明有 2 个苹果,又得到 3 个,现在一共有几个?\n请在最后一行用 #### 数字 给出答案。"}] encoded = tokenizer.apply_chat_template( messages, tokenize=True, add_generation_prompt=True, ) # 不传 enable_thinking,保留 checkpoint 聊天模板的默认行为。 token_ids = token_ids_from_template(encoded) max_new_tokens = 1024 if len(token_ids) + max_new_tokens > 4096: raise ValueError("提示长度加生成预算超过 4096;本示例不自动截断输入") sampling = SamplingParams( temperature=0.0, top_p=1.0, top_k=-1, n=1, max_tokens=max_new_tokens, seed=42, ignore_eos=False, stop_token_ids=stop_ids, repetition_penalty=generation.get("repetition_penalty", 1.0), min_tokens=generation.get("min_new_tokens") or 0, presence_penalty=0.0, frequency_penalty=0.0, skip_special_tokens=True, ) outputs = llm.generate( [{"prompt_token_ids": token_ids}], sampling_params=sampling, use_tqdm=False, ) print(tokenizer.decode(outputs[0].outputs[0].token_ids, skip_special_tokens=True)) if __name__ == "__main__": main()