import torchfrom transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig model_id = "your-username/moonlight-qwen3-4b-v7" # 4-bit Quantization (fits in 3.5 GB VRAM)quant_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.float16, bnb_4bit_use_double_quant=True,) tokenizer = AutoTokenizer.from_pretrained(model_id)model = AutoModelForCausalLM.from_pretrained( model_id, quantization_config=quant_config, device_map="auto", torch_dtype=torch.float16,) system_prompt = """You are Moonlight, a privacy-first AI assistant powered by Qwen3-4B.Be accurate, helpful, practical, and concise.Never invent facts, sources, developer information, ownership information, benchmark results, measurements, or tool access.""" messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": "What are the core design principles of Moonlight AI?"},] inputs = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").to(model.device) outputs = model.generate(inputs, max_new_tokens=256, do_sample=False)response = tokenizer.decode(outputs[0][inputs.shape[1]:], skip_special_tokens=True)print(response)