GLM-5.3 is live. Run Z.ai's latest model on Friendli Model APIs. Try it today ➜
RedHatAI
Available on FriendliAI
Run this model inference on single tenant GPU with unmatched speed and reliability at scale.
Model Details
Model Provider
RedHatAI
Model Tree
Qwen/Qwen3.8-27B
Input Modalities
Output Modalities
Supported Functionality
This model is a quantized version of Qwen/Qwen3.8-27B. It was evaluated on several tasks to assess its quality in comparison to the unquantized model.
This model was obtained by quantizing the weights of Qwen/Qwen3.8-27B to INT4 data type while keeping activations in original precision, with FP8 KV cache, ready for inference with vLLM.
This optimization reduces the number of bits per parameter from 16 to 4, reducing the disk size and GPU memory requirements by approximately 75%.
Only the weights of the linear operators within transformer blocks are quantized using LLM Compressor.
bash
vllm serve RedHatAI/Qwen3.8-27B-INT4 \ --reasoning-parser qwen3 \ --enable-auto-tool-choice \ --tool-call-parser qwen3_xml \ --speculative-config '{"method":"mtp","num_speculative_tokens":3}'For optimal peformance, consider using the DSpark draft model RedHatAI/Qwen3.8-27B-speculator.dspark for speculative decoding, shown below.
bash
vllm serve RedHatAI/Qwen3.8-27B-INT4 \ --enable-auto-tool-choice \ --tool-call-parser qwen3_xml \ --reasoning-parser qwen3 \ --speculative-config '{"model":"RedHatAI/Qwen3.8-27B-speculator.dspark","num_speculative_tokens":7,"method":"dspark"}'This model was created by applying LLM Compressor with calibration samples from open-perfectblend, using AWQ smoothing and the W4A16 GPTQ scheme, exported in compressed-tensors format.
python
import torchfrom datasets import load_datasetfrom transformers import AutoProcessor, Qwen3_5ForConditionalGeneration from llmcompressor import oneshotfrom llmcompressor.modifiers.gptq import GPTQModifierfrom llmcompressor.modifiers.transform.awq import AWQModifierfrom llmcompressor.utils import load_context MODEL_ID = "Qwen/Qwen3.8-27B" # Load model.with load_context(Qwen3_5ForConditionalGeneration): model = Qwen3_5ForConditionalGeneration.from_pretrained(MODEL_ID)processor = AutoProcessor.from_pretrained(MODEL_ID) recipe = [ AWQModifier(duo_scaling="both"), GPTQModifier( targets="Linear", scheme="W4A16", ignore=[ "re:visual.*", "re:model.visual.*", r"re:.*lm_head", "re:.*embed_tokens$", r"re:.*linear_attn\.in_proj_a$", r"re:.*linear_attn\.in_proj_b$", ], kv_cache_scheme={ "num_bits": 8, "type": "float", "symmetric": True, "strategy": "tensor", "dynamic": False, "observer": "static_minmax", }, ),] NUM_CALIBRATION_SAMPLES = 512MAX_SEQUENCE_LENGTH = 4096 ds = load_dataset( "mlabonne/open-perfectblend", split=f"train[:{NUM_CALIBRATION_SAMPLES}]",)ds = ds.shuffle(seed=42) ROLE_MAP = {"human": "user", "gpt": "assistant"} def preprocess_function(example): messages = [ { "role": ROLE_MAP.get(msg["from"], msg["from"]), "content": [{"type": "text", "text": msg["value"]}], } for msg in example["conversations"] ] return processor.apply_chat_template( messages, tokenize=True, return_dict=True, add_generation_prompt=False, processor_kwargs={ "return_tensors": "pt", "padding": False, "truncation": True, "max_length": MAX_SEQUENCE_LENGTH, "add_special_tokens": False, }, ) ds = ds.map(preprocess_function, batched=False, remove_columns=ds.column_names) def data_collator(batch): assert len(batch) == 1 return {key: torch.tensor(value) for key, value in batch[0].items()} # Apply quantization.oneshot( model=model, recipe=recipe, dataset=ds, max_seq_length=MAX_SEQUENCE_LENGTH, num_calibration_samples=NUM_CALIBRATION_SAMPLES, moe_calibrate_all_experts=True, data_collator=data_collator,) # Save to disk in compressed-tensors format.SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-INT4"model.save_pretrained(SAVE_DIR)processor.save_pretrained(SAVE_DIR)This model was evaluated on GSM8K Platinum, IFEval, MMLU-Pro, MATH-500, GPQA Diamond, and AIME 2025 using lm-evaluation-harness and lighteval, all served with vLLM (OpenAI-compatible API). Each benchmark was run with 3 seeds (1234, 2345, 3456; 8 seeds for AIME 2025) and the results averaged; recovery is computed against the BF16 model.
vllm serve RedHatAI/Qwen3.8-27B-INT4 \ --reasoning-parser qwen3 \ --enable-auto-tool-choice \ --tool-call-parser qwen3_xml \ --speculative-config '{"method":"mtp","num_speculative_tokens":3}'vllm serve RedHatAI/Qwen3.8-27B-INT4 \ --enable-auto-tool-choice \ --tool-call-parser qwen3_xml \ --reasoning-parser qwen3 \ --speculative-config '{"model":"RedHatAI/Qwen3.8-27B-speculator.dspark","num_speculative_tokens":7,"method":"dspark"}'import torchfrom datasets import load_datasetfrom transformers import AutoProcessor, Qwen3_5ForConditionalGeneration from llmcompressor import oneshotfrom llmcompressor.modifiers.gptq import GPTQModifierfrom llmcompressor.modifiers.transform.awq import AWQModifierfrom llmcompressor.utils import load_context MODEL_ID = "Qwen/Qwen3.8-27B" # Load model.with load_context(Qwen3_5ForConditionalGeneration): model = Qwen3_5ForConditionalGeneration.from_pretrained(MODEL_ID)processor = AutoProcessor.from_pretrained(MODEL_ID) recipe = [ AWQModifier(duo_scaling="both"), GPTQModifier( targets="Linear", scheme="W4A16", ignore=[ "re:visual.*", "re:model.visual.*", r"re:.*lm_head", "re:.*embed_tokens$", r"re:.*linear_attn\.in_proj_a$", r"re:.*linear_attn\.in_proj_b$", ], kv_cache_scheme={ "num_bits": 8, "type": "float", "symmetric": True, "strategy": "tensor", "dynamic": False, "observer": "static_minmax", }, ),] NUM_CALIBRATION_SAMPLES = 512MAX_SEQUENCE_LENGTH = 4096 ds = load_dataset( "mlabonne/open-perfectblend", split=f"train[:{NUM_CALIBRATION_SAMPLES}]",)ds = ds.shuffle(seed=42) ROLE_MAP = {"human": "user", "gpt": "assistant"} def preprocess_function(example): messages = [ { "role": ROLE_MAP.get(msg["from"], msg["from"]), "content": [{"type": "text", "text": msg["value"]}], } for msg in example["conversations"] ] return processor.apply_chat_template( messages, tokenize=True, return_dict=True, add_generation_prompt=False, processor_kwargs={ "return_tensors": "pt", "padding": False, "truncation": True, "max_length": MAX_SEQUENCE_LENGTH, "add_special_tokens": False, }, ) ds = ds.map(preprocess_function, batched=False, remove_columns=ds.column_names) def data_collator(batch): assert len(batch) == 1 return {key: torch.tensor(value) for key, value in batch[0].items()} # Apply quantization.oneshot( model=model, recipe=recipe, dataset=ds, max_seq_length=MAX_SEQUENCE_LENGTH, num_calibration_samples=NUM_CALIBRATION_SAMPLES, moe_calibrate_all_experts=True, data_collator=data_collator,) # Save to disk in compressed-tensors format.SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-INT4"model.save_pretrained(SAVE_DIR)processor.save_pretrained(SAVE_DIR)