GLM-5.3 is live. Run Z.ai's latest model on Friendli Model APIs. Try it today ➜
inference-optimization
Available on FriendliAI
Run this model inference on single tenant GPU with unmatched speed and reliability at scale.
Run this model inference with full control and performance in your environment.
Model Details
Model Provider
inference-optimization
Model Tree
Input Modalities
Output Modalities
Supported Functionality
Uses Transformers 5.17.0 and LLM Compressor PR #3225.
python
import torchfrom compressed_tensors.offload import set_onload_devicefrom compressed_tensors.quantization import preset_name_to_schemefrom transformers import AutoTokenizer, Glm4MoeForCausalLM from llmcompressor import oneshotfrom llmcompressor.modifiers.quantization import QuantizationModifierfrom llmcompressor.utils import load_context MODEL_ID = "inference-optimization/GLM-4.5-Air-0.72B-MTP"SAVE_DIR = "GLM-4.5-Air-0.72B-MTP-FP8-Dynamic" with load_context(Glm4MoeForCausalLM, load_mtp=True): model = Glm4MoeForCausalLM.from_pretrained( MODEL_ID, dtype=torch.bfloat16, device_map="cpu", )set_onload_device(model, "cuda") recipe = QuantizationModifier( config_groups={ "mtp": preset_name_to_scheme("FP8_DYNAMIC", targets=[r"re:^mtp\.layers\."]), "backbone": preset_name_to_scheme("FP8_DYNAMIC", targets=["Linear"]), }, ignore=["lm_head", r"re:.*\.eh_proj$", r"re:.*\.indexer\..*"],) oneshot(model=model, recipe=recipe)model.save_pretrained(SAVE_DIR)AutoTokenizer.from_pretrained(MODEL_ID).save_pretrained(SAVE_DIR)