import torchfrom transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfigfrom peft import PeftModel bnb_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=True, bnb_4bit_compute_dtype=torch.float16, # not bfloat16 -- see note below if on a Turing GPU (e.g. T4)) tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-8B")base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-8B", quantization_config=bnb_config, device_map="auto")model = PeftModel.from_pretrained(base, "code-aitazaz/roman-urdu-qlora-qwen3-8b") messages = [{"role": "user", "content": "Pakistan ka dar-ul-hukumat kya hai?"}]prompt_ids = tokenizer.apply_chat_template( messages, tokenize=True, add_generation_prompt=True, enable_thinking=False, return_tensors="pt")["input_ids"].to(model.device) output_ids = model.generate( prompt_ids, max_new_tokens=256, do_sample=False, repetition_penalty=1.2, # see "Known limitation" below -- important no_repeat_ngram_size=3, pad_token_id=tokenizer.pad_token_id or tokenizer.eos_token_id,)print(tokenizer.decode(output_ids[0][prompt_ids.shape[1]:], skip_special_tokens=True))