GLM-5.3 is live. Run Z.ai's latest model on Friendli Model APIs. Try it today ➜
amd
Available on FriendliAI
Run this model inference on single tenant GPU with unmatched speed and reliability at scale.
Model Details
Model Provider
amd
Model Tree
unsloth/gpt-oss-20b-BF16
Input Modalities
Output Modalities
Supported Functionality
This is a quantized version of gpt-oss-20b-BF16 created by AMD using LLM Compressor (compressed-tensors) for ZenDNN-optimized CPU inference.
The model was quantized from gpt-oss-20b-BF16 using LLM Compressor via the GPTQ algorithm. This reduces the model weights from 39.0 GiB to 12.0 GiB on disk (~69% reduction).
compressed-tensors, num_bits=4, type=int, symmetric=true, group_size=64group_size=64, actorder=static), stored as pack-quantizedw13 gate+up and w2 down projections) and self_attn.{q,k,v,o}_projmlp.router weight and bias), the attention sinks (self_attn.sinks), every Linear bias, lm_head, embed_tokens, and the layer normsgpt-oss stores each layer's 32 experts as fused 3D tensors, which a targets=["Linear"] recipe cannot see. Loading inside load_context() together with the gptoss_linear_experts shim exposes them as per-expert Linear submodules, so the quantizer reaches every expert projection. The router is skipped because it is a tiny Linear whose logits decide expert assignment, where a 4-bit rounding error can flip the top-4 selection and change which experts run.
python
import gptoss_linear_experts # noqa: F401 (registers GptOssLinearExperts)from compressed_tensors.quantization import ( QuantizationArgs, QuantizationScheme, QuantizationStrategy, QuantizationType,)from datasets import load_datasetfrom transformers import AutoModelForCausalLM, AutoTokenizer from llmcompressor import oneshotfrom llmcompressor.modifiers.gptq import GPTQModifierfrom llmcompressor.utils import load_context MODEL_ID = "unsloth/gpt-oss-20b-BF16"SAVE_DIR = "./gpt-oss-20b-BF16-w4a16-llmcompressor-v0.13.0"CALIB_SIZE = 128MAX_SEQ_LENGTH = 2048# gpt-oss layers are 2880 wide, which 128 does not divide.GROUP_SIZE = 64 IGNORE = [ "lm_head", r"re:.*\.router$", r"re:.*\.router\..*", r"re:.*\.gate$", r"re:.*\.mlp\.gate$",] def _to_text(example: dict) -> dict: parts = [m.get("content", "") for m in example.get("messages", [])] return {"text": "\n".join(p for p in parts if p)} # Step 1: Load the BF16 model inside load_context(), which linearizes the fused# 3D expert tensors into per-expert Linear submodules.print(f"loading {MODEL_ID}", flush=True)with load_context(): model = AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype="bfloat16")tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) experts = model.model.layers[0].mlp.expertsprint("linearized experts class:", type(experts).__name__, flush=True) # Step 2: Load calibration data. GPTQ is data-driven: it needs real activations# to build the per-layer Hessians used to compensate the rounding error.print(f"loading {CALIB_SIZE} calibration samples", flush=True)calib = load_dataset("HuggingFaceH4/ultrachat_200k", split=f"train_sft[:{CALIB_SIZE}]")calib = calib.map(_to_text, remove_columns=calib.column_names, desc="text")if not getattr(tokenizer, "pad_token", None): tokenizer.pad_token = tokenizer.eos_tokencalib = calib.map( lambda e: tokenizer( e["text"], truncation=True, max_length=MAX_SEQ_LENGTH, add_special_tokens=False, ), remove_columns=["text"], desc="tokenize",) # Step 3: Build the W4A16 recipe with an explicit group size instead of the# scheme="W4A16" preset, which would imply the unusable group_size=128.weights = QuantizationArgs( num_bits=4, type=QuantizationType.INT, symmetric=True, strategy=QuantizationStrategy.GROUP, group_size=GROUP_SIZE,)recipe = GPTQModifier( targets="Linear", config_groups={"group_0": QuantizationScheme(targets=["Linear"], weights=weights)}, ignore=IGNORE,) # Step 4: One-shot quantize with calibration and save in compressed-tensors format.print("quantizing with GPTQ (W4A16, group 64)", flush=True)oneshot(model=model, dataset=calib, recipe=recipe, max_seq_length=MAX_SEQ_LENGTH) print(f"saving to {SAVE_DIR}", flush=True)model.save_pretrained(SAVE_DIR, save_compressed=True)tokenizer.save_pretrained(SAVE_DIR)print("done", flush=True)python
from vllm import LLM, SamplingParams model = LLM( model="amd/gpt-oss-20b-BF16-w4a16-llmcompressor", dtype="bfloat16",) sampling_params = SamplingParams(temperature=0.7, max_tokens=256)outputs = model.generate(["Hello, how are you?"], sampling_params)print(outputs[0].outputs[0].text)text
torch==2.13.0.0zentorch==2.13.0.0vllm==0.28.0llmcompressor==0.13.0For optimal performance, set LD_PRELOAD with libomp.so (LLVM OpenMP) or libiomp5.so (Intel OpenMP):
bash
# Using LLVM OpenMP (llvmopenmp)export LD_PRELOAD=$(find /path/to/env -name "libomp.so" | head -1) # Or using Intel OpenMP (libiomp)export LD_PRELOAD=$(find /path/to/env -name "libiomp5.so" | head -1)Note: Set
LD_PRELOADbefore launching vLLM or any inference script.
The model was evaluated against the BF16 (unquantized) baseline on standard benchmarks using lm-evaluation-harness with the vLLM engine.
| Benchmark | BF16 Baseline | W4A16 (this model) | Recovery |
|---|---|---|---|
| GSM8K (5-shot) | 0.8741 | 0.8643 | 98.88% |
bash
lm_eval \ --model vllm \ --model_args pretrained=amd/gpt-oss-20b-BF16-w4a16-llmcompressor,dtype=bfloat16,max_model_len=4096 \ --tasks gsm8k \ --batch_size auto \ --trust_remote_code \ --num_fewshot 5 \ --apply_chat_template \ --log_samples \ --gen_kwargs "max_gen_toks=2048" \ --output_path .This model is distributed under the same license as the source model. See the LICENSE file for details.
Modifications copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved.
import gptoss_linear_experts # noqa: F401 (registers GptOssLinearExperts)from compressed_tensors.quantization import ( QuantizationArgs, QuantizationScheme, QuantizationStrategy, QuantizationType,)from datasets import load_datasetfrom transformers import AutoModelForCausalLM, AutoTokenizer from llmcompressor import oneshotfrom llmcompressor.modifiers.gptq import GPTQModifierfrom llmcompressor.utils import load_context MODEL_ID = "unsloth/gpt-oss-20b-BF16"SAVE_DIR = "./gpt-oss-20b-BF16-w4a16-llmcompressor-v0.13.0"CALIB_SIZE = 128MAX_SEQ_LENGTH = 2048# gpt-oss layers are 2880 wide, which 128 does not divide.GROUP_SIZE = 64 IGNORE = [ "lm_head", r"re:.*\.router$", r"re:.*\.router\..*", r"re:.*\.gate$", r"re:.*\.mlp\.gate$",] def _to_text(example: dict) -> dict: parts = [m.get("content", "") for m in example.get("messages", [])] return {"text": "\n".join(p for p in parts if p)} # Step 1: Load the BF16 model inside load_context(), which linearizes the fused# 3D expert tensors into per-expert Linear submodules.print(f"loading {MODEL_ID}", flush=True)with load_context(): model = AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype="bfloat16")tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) experts = model.model.layers[0].mlp.expertsprint("linearized experts class:", type(experts).__name__, flush=True) # Step 2: Load calibration data. GPTQ is data-driven: it needs real activations# to build the per-layer Hessians used to compensate the rounding error.print(f"loading {CALIB_SIZE} calibration samples", flush=True)calib = load_dataset("HuggingFaceH4/ultrachat_200k", split=f"train_sft[:{CALIB_SIZE}]")calib = calib.map(_to_text, remove_columns=calib.column_names, desc="text")if not getattr(tokenizer, "pad_token", None): tokenizer.pad_token = tokenizer.eos_tokencalib = calib.map( lambda e: tokenizer( e["text"], truncation=True, max_length=MAX_SEQ_LENGTH, add_special_tokens=False, ), remove_columns=["text"], desc="tokenize",) # Step 3: Build the W4A16 recipe with an explicit group size instead of the# scheme="W4A16" preset, which would imply the unusable group_size=128.weights = QuantizationArgs( num_bits=4, type=QuantizationType.INT, symmetric=True, strategy=QuantizationStrategy.GROUP, group_size=GROUP_SIZE,)recipe = GPTQModifier( targets="Linear", config_groups={"group_0": QuantizationScheme(targets=["Linear"], weights=weights)}, ignore=IGNORE,) # Step 4: One-shot quantize with calibration and save in compressed-tensors format.print("quantizing with GPTQ (W4A16, group 64)", flush=True)oneshot(model=model, dataset=calib, recipe=recipe, max_seq_length=MAX_SEQ_LENGTH) print(f"saving to {SAVE_DIR}", flush=True)model.save_pretrained(SAVE_DIR, save_compressed=True)tokenizer.save_pretrained(SAVE_DIR)print("done", flush=True)from vllm import LLM, SamplingParams model = LLM( model="amd/gpt-oss-20b-BF16-w4a16-llmcompressor", dtype="bfloat16",) sampling_params = SamplingParams(temperature=0.7, max_tokens=256)outputs = model.generate(["Hello, how are you?"], sampling_params)print(outputs[0].outputs[0].text)torch==2.13.0.0zentorch==2.13.0.0vllm==0.28.0llmcompressor==0.13.0# Using LLVM OpenMP (llvmopenmp)export LD_PRELOAD=$(find /path/to/env -name "libomp.so" | head -1) # Or using Intel OpenMP (libiomp)export LD_PRELOAD=$(find /path/to/env -name "libiomp5.so" | head -1)lm_eval \ --model vllm \ --model_args pretrained=amd/gpt-oss-20b-BF16-w4a16-llmcompressor,dtype=bfloat16,max_model_len=4096 \ --tasks gsm8k \ --batch_size auto \ --trust_remote_code \ --num_fewshot 5 \ --apply_chat_template \ --log_samples \ --gen_kwargs "max_gen_toks=2048" \ --output_path .