from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import PeftModel
import torch
bnb_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_quant_type="nf4",
bnb_4bit_compute_dtype=torch.float16,
)
base_model = AutoModelForCausalLM.from_pretrained(
"Qwen/Qwen2.5-0.5B",
quantization_config=bnb_config,
device_map="auto",
)
model = PeftModel.from_pretrained(base_model, "ProfRutPatel/Qwen2.5-0.5B-India-Propaganda-Detector")
tokenizer = AutoTokenizer.from_pretrained("ProfRutPatel/Qwen2.5-0.5B-India-Propaganda-Detector")
system_prompt = "You are an expert analyst specialized in detecting anti-India propaganda..."
text = "Your text to analyze here"
prompt = f"<|im_start|>system\n{system_prompt}<|im_end|>\n<|im_start|>user\nAnalyze the following text for anti-India propaganda:\n\n\"{text}\"<|im_end|>\n<|im_start|>assistant\n"
inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
outputs = model.generate(**inputs, max_new_tokens=200, temperature=0.1)
result = tokenizer.decode(outputs[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True)
print(result)