GLM-5.3 is live. Run Z.ai's latest model on Friendli Model APIs. Try it today ➜
lingshu-medical-mllm
Available on FriendliAI
Run this model inference on single tenant GPU with unmatched speed and reliability at scale.
Model Details
Model Provider
lingshu-medical-mllm
Model Tree
Input Modalities
Output Modalities
Supported Functionality
python
from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessorfrom qwen_vl_utils import process_vision_info # We recommend enabling flash_attention_2 for better acceleration and memory saving, especially in multi-image and video scenarios.model = Qwen2_5_VLForConditionalGeneration.from_pretrained( "lingshu-medical-mllm/Lingshu-7B", torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2", device_map="auto",) processor = AutoProcessor.from_pretrained("lingshu-medical-mllm/Lingshu-7B") messages = [ { "role": "user", "content": [ { "type": "image", "image": "example.png", }, {"type": "text", "text": "Describe this image."}, ], }] # Preparation for inferencetext = processor.apply_chat_template( messages, tokenize=False, add_generation_prompt=True)image_inputs, video_inputs = process_vision_info(messages)inputs = processor( text=[text], images=image_inputs, videos=video_inputs, padding=True, return_tensors="pt",)inputs = inputs.to(model.device) # Inference: Generation of the outputgenerated_ids = model.generate(**inputs, max_new_tokens=128)generated_ids_trimmed = [ out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]output_text = processor.batch_decode( generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)print(output_text)python
from vllm import LLM, SamplingParamsfrom qwen_vl_utils import process_vision_infoimport PILfrom transformers import AutoProcessor processor = AutoProcessor.from_pretrained("lingshu-medical-mllm/Lingshu-7B")llm = LLM(model="lingshu-medical-mllm/Lingshu-7B", limit_mm_per_prompt = {"image": 4}, tensor_parallel_size=2, enforce_eager=True, trust_remote_code=True,)sampling_params = SamplingParams( temperature=0.7, top_p=1, repetition_penalty=1, max_tokens=1024, stop_token_ids=[], ) text = "What does the image show?"image_path = "example.png"image = PIL.Image.open(image_path) message = [ { "role":"user", "content":[ {"type":"image","image":image}, {"type":"text","text":text} ] }]prompt = processor.apply_chat_template( message, tokenize=False, add_generation_prompt=True,)image_inputs, video_inputs = process_vision_info(message)mm_data = {}mm_data["image"] = image_inputsprocessed_input = { "prompt": prompt, "multi_modal_data": mm_data,} outputs = llm.generate([processed_input], sampling_params=sampling_params)print(outputs[0].outputs[0].text)If you find our project useful, we hope you would kindly star our repo and cite our work as follows:
text
@article{xu2025lingshu, title={Lingshu: A Generalist Foundation Model for Unified Multimodal Medical Understanding and Reasoning}, author={Xu, Weiwen and Chan, Hou Pong and Li, Long and Aljunied, Mahani and Yuan, Ruifeng and Wang, Jianyu and Xiao, Chenghao and Chen, Guizhen and Liu, Chaoqun and Li, Zhaodonghui and others}, journal={arXiv preprint arXiv:2506.07044}, year={2025}}from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessorfrom qwen_vl_utils import process_vision_info # We recommend enabling flash_attention_2 for better acceleration and memory saving, especially in multi-image and video scenarios.model = Qwen2_5_VLForConditionalGeneration.from_pretrained( "lingshu-medical-mllm/Lingshu-7B", torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2", device_map="auto",) processor = AutoProcessor.from_pretrained("lingshu-medical-mllm/Lingshu-7B") messages = [ { "role": "user", "content": [ { "type": "image", "image": "example.png", }, {"type": "text", "text": "Describe this image."}, ], }] # Preparation for inferencetext = processor.apply_chat_template( messages, tokenize=False, add_generation_prompt=True)image_inputs, video_inputs = process_vision_info(messages)inputs = processor( text=[text], images=image_inputs, videos=video_inputs, padding=True, return_tensors="pt",)inputs = inputs.to(model.device) # Inference: Generation of the outputgenerated_ids = model.generate(**inputs, max_new_tokens=128)generated_ids_trimmed = [ out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]output_text = processor.batch_decode( generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)print(output_text)from vllm import LLM, SamplingParamsfrom qwen_vl_utils import process_vision_infoimport PILfrom transformers import AutoProcessor processor = AutoProcessor.from_pretrained("lingshu-medical-mllm/Lingshu-7B")llm = LLM(model="lingshu-medical-mllm/Lingshu-7B", limit_mm_per_prompt = {"image": 4}, tensor_parallel_size=2, enforce_eager=True, trust_remote_code=True,)sampling_params = SamplingParams( temperature=0.7, top_p=1, repetition_penalty=1, max_tokens=1024, stop_token_ids=[], ) text = "What does the image show?"image_path = "example.png"image = PIL.Image.open(image_path) message = [ { "role":"user", "content":[ {"type":"image","image":image}, {"type":"text","text":text} ] }]prompt = processor.apply_chat_template( message, tokenize=False, add_generation_prompt=True,)image_inputs, video_inputs = process_vision_info(message)mm_data = {}mm_data["image"] = image_inputsprocessed_input = { "prompt": prompt, "multi_modal_data": mm_data,} outputs = llm.generate([processed_input], sampling_params=sampling_params)print(outputs[0].outputs[0].text)@article{xu2025lingshu, title={Lingshu: A Generalist Foundation Model for Unified Multimodal Medical Understanding and Reasoning}, author={Xu, Weiwen and Chan, Hou Pong and Li, Long and Aljunied, Mahani and Yuan, Ruifeng and Wang, Jianyu and Xiao, Chenghao and Chen, Guizhen and Liu, Chaoqun and Li, Zhaodonghui and others}, journal={arXiv preprint arXiv:2506.07044}, year={2025}}