GLM-5.3 is live. Run Z.ai's latest model on Friendli Model APIs. Try it today ➜
RedHatAI
Available on FriendliAI
Run this model inference on single tenant GPU with unmatched speed and reliability at scale.
Run this model inference with full control and performance in your environment.
Model Details
Model Provider
RedHatAI
Model Tree
openai/whisper-large-v3
Input Modalities
Output Modalities
Supported Functionality
Quantized version of openai/whisper-large-v3.
This model was obtained by quantizing the weights of openai/whisper-large-v3 to FP8 data type, ready for inference with vLLM >= 0.5.2.
This model can be deployed efficiently using the vLLM backend, as shown in the example below.
python
from vllm.assets.audio import AudioAssetfrom vllm import LLM, SamplingParams # prepare modelllm = LLM( model="neuralmagic/whisper-large-v3-FP8-Dynamic", max_model_len=448, max_num_seqs=400, limit_mm_per_prompt={"audio": 1},) # prepare inputsinputs = { # Test explicit encoder/decoder prompt "encoder_prompt": { "prompt": "", "multi_modal_data": { "audio": AudioAsset("winning_call").audio_and_sample_rate, }, }, "decoder_prompt": "<|startoftranscript|>",} # generate responseprint("========== SAMPLE GENERATION ==============")outputs = llm.generate(inputs, SamplingParams(temperature=0.0, max_tokens=64))print(f"PROMPT : {outputs[0].prompt}")print(f"RESPONSE: {outputs[0].outputs[0].text}")print("==========================================")vLLM also supports OpenAI-compatible serving. See the documentation for more details.
This model was created with llm-compressor by running the code snippet below.
bash
python quantize.py \ --model_path openai/whisper-large-v3 \ --quant_path output_dir/whisper-large-v3-FP8-Dynamicpython
import argparseimport torchimport osfrom datasets import load_datasetfrom transformers import WhisperProcessorfrom llmcompressor import oneshotfrom llmcompressor.modifiers.quantization import QuantizationModifierfrom llmcompressor.transformers.tracing import TraceableWhisperForConditionalGenerationfrom compressed_tensors.quantization import QuantizationType # --- Args ---parser = argparse.ArgumentParser()parser.add_argument('--model_path', type=str, required=True)parser.add_argument('--quant_path', type=str, required=True)parser.add_argument('--observer', type=str, default="minmax")args = parser.parse_args() # --- Load Model ---model = TraceableWhisperForConditionalGeneration.from_pretrained( args.model_path, device_map="auto", torch_dtype="auto",)model.config.forced_decoder_ids = Noneprocessor = WhisperProcessor.from_pretrained(args.model_path) # --- Recipe (FP8 Dynamic) ---recipe = [ QuantizationModifier( targets="Linear", scheme="FP8_DYNAMIC", sequential_targets=["WhisperEncoderLayer", "WhisperDecoderLayer"], ignore=["re:.*lm_head"], )] # --- Run oneshot ---oneshot( model=model, recipe=recipe, trust_remote_code_model=True,) # --- Save ---os.makedirs(args.quant_path, exist_ok=True)model.save_pretrained(args.quant_path, save_compressed=True)processor.save_pretrained(args.quant_path)The model was evaluated on LibriSpeech and Fleurs datasets using lmms-eval, via the following commands:
Librispeech:
text
lmms-eval \ --model=whisper_vllm \ --model_args="pretrained=neuralmagic-ent/whisper-large-v3-FP8-Dynamic" \ --batch_size 64 \ --output_path <output_file_path> \ --tasks librispeechFleurs:
text
lmms-eval \ --model=whisper_vllm \ --model_args="pretrained=neuralmagic-ent/whisper-large-v3-FP8-Dynamic" \ --batch_size 64 \ --output_path <output_file_path> \ --tasks fleurspython quantize.py \ --model_path openai/whisper-large-v3 \ --quant_path output_dir/whisper-large-v3-FP8-Dynamiclmms-eval \ --model=whisper_vllm \ --model_args="pretrained=neuralmagic-ent/whisper-large-v3-FP8-Dynamic" \ --batch_size 64 \ --output_path <output_file_path> \ --tasks librispeechlmms-eval \ --model=whisper_vllm \ --model_args="pretrained=neuralmagic-ent/whisper-large-v3-FP8-Dynamic" \ --batch_size 64 \ --output_path <output_file_path> \ --tasks fleursfrom vllm.assets.audio import AudioAssetfrom vllm import LLM, SamplingParams # prepare modelllm = LLM( model="neuralmagic/whisper-large-v3-FP8-Dynamic", max_model_len=448, max_num_seqs=400, limit_mm_per_prompt={"audio": 1},) # prepare inputsinputs = { # Test explicit encoder/decoder prompt "encoder_prompt": { "prompt": "", "multi_modal_data": { "audio": AudioAsset("winning_call").audio_and_sample_rate, }, }, "decoder_prompt": "<|startoftranscript|>",} # generate responseprint("========== SAMPLE GENERATION ==============")outputs = llm.generate(inputs, SamplingParams(temperature=0.0, max_tokens=64))print(f"PROMPT : {outputs[0].prompt}")print(f"RESPONSE: {outputs[0].outputs[0].text}")print("==========================================")import argparseimport torchimport osfrom datasets import load_datasetfrom transformers import WhisperProcessorfrom llmcompressor import oneshotfrom llmcompressor.modifiers.quantization import QuantizationModifierfrom llmcompressor.transformers.tracing import TraceableWhisperForConditionalGenerationfrom compressed_tensors.quantization import QuantizationType # --- Args ---parser = argparse.ArgumentParser()parser.add_argument('--model_path', type=str, required=True)parser.add_argument('--quant_path', type=str, required=True)parser.add_argument('--observer', type=str, default="minmax")args = parser.parse_args() # --- Load Model ---model = TraceableWhisperForConditionalGeneration.from_pretrained( args.model_path, device_map="auto", torch_dtype="auto",)model.config.forced_decoder_ids = Noneprocessor = WhisperProcessor.from_pretrained(args.model_path) # --- Recipe (FP8 Dynamic) ---recipe = [ QuantizationModifier( targets="Linear", scheme="FP8_DYNAMIC", sequential_targets=["WhisperEncoderLayer", "WhisperDecoderLayer"], ignore=["re:.*lm_head"], )] # --- Run oneshot ---oneshot( model=model, recipe=recipe, trust_remote_code_model=True,) # --- Save ---os.makedirs(args.quant_path, exist_ok=True)model.save_pretrained(args.quant_path, save_compressed=True)processor.save_pretrained(args.quant_path)