GLM-5.3 is live. Run Z.ai's latest model on Friendli Model APIs. Try it today ➜
lucasoc
Available on FriendliAI
Run this model inference on single tenant GPU with unmatched speed and reliability at scale.
Run this model inference with full control and performance in your environment.
Model Details
Model Provider
lucasoc
Model Tree
Qwen/Qwen2.5-VL-3B-Instruct
Input Modalities
Output Modalities
Supported Functionality
| Metric | Base Model (Zero-Shot) | Standard SFT (Cross-Entropy) | Our LoRA (ICDAR Metric Loss) |
|---|---|---|---|
| Valid Markdown Table | 88.74% | 99.73% | 99.73% |
| Cell Precision | 21.08% | 21.37% | 39.58% (+85.2%) |
| Cell F1-Score | 18.42% | 20.60% | 34.67% (+68.3%) |
| TEDS (Tree Edit Distance) | 23.16% | 23.16% | 36.24% (+56.5%) |
| ICDAR Numerical RMS | 48.12% | 48.12% | 51.66% (+7.4%) |
| ICDAR Composite Score | 35.64% | 35.64% | 43.95% (+23.3%) |
bash
pip install torch transformers peft qwen-vl-utils pillow torchvision bitsandbytespython
import torchfrom PIL import Imagefrom transformers import AutoProcessor, Qwen2_5_VLForConditionalGeneration, BitsAndBytesConfigfrom peft import PeftModelfrom qwen_vl_utils import process_vision_info model_id = "Qwen/Qwen2.5-VL-3B-Instruct"adapter_id = "lucasoc/sci-image-markdown-qwen2.5-vl-3b" # 1. Load processorprocessor = AutoProcessor.from_pretrained(adapter_id) # 2. Load 4-bit base modelbnb_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.float16, bnb_4bit_use_double_quant=True,)base_model = Qwen2_5_VLForConditionalGeneration.from_pretrained( model_id, quantization_config=bnb_config, torch_dtype=torch.float16, device_map="auto") # 3. Load trained LoRA adaptermodel = PeftModel.from_pretrained(base_model, adapter_id)model.eval() # 4. Extract table from figureimage_path = "figure.png"image = Image.open(image_path).convert("RGB") messages = [ { "role": "user", "content": [ {"type": "image", "image": image}, {"type": "text", "text": "Extract the plotted quantitative data into a clean Markdown table with column headers."} ] }] text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)image_inputs, _ = process_vision_info(messages)inputs = processor(text=[text], images=image_inputs, padding=True, return_tensors="pt").to("cuda") with torch.inference_mode(): generated_ids = model.generate(**inputs, max_new_tokens=1024, temperature=0.0) generated_ids_trimmed = [ out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids) ] output_text = processor.batch_decode( generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False )[0] print("--- Extracted Markdown Table ---")print(output_text)Qwen2.5-VL-3B-Instructpip install torch transformers peft qwen-vl-utils pillow torchvision bitsandbytesimport torchfrom PIL import Imagefrom transformers import AutoProcessor, Qwen2_5_VLForConditionalGeneration, BitsAndBytesConfigfrom peft import PeftModelfrom qwen_vl_utils import process_vision_info model_id = "Qwen/Qwen2.5-VL-3B-Instruct"adapter_id = "lucasoc/sci-image-markdown-qwen2.5-vl-3b" # 1. Load processorprocessor = AutoProcessor.from_pretrained(adapter_id) # 2. Load 4-bit base modelbnb_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.float16, bnb_4bit_use_double_quant=True,)base_model = Qwen2_5_VLForConditionalGeneration.from_pretrained( model_id, quantization_config=bnb_config, torch_dtype=torch.float16, device_map="auto") # 3. Load trained LoRA adaptermodel = PeftModel.from_pretrained(base_model, adapter_id)model.eval() # 4. Extract table from figureimage_path = "figure.png"image = Image.open(image_path).convert("RGB") messages = [ { "role": "user", "content": [ {"type": "image", "image": image}, {"type": "text", "text": "Extract the plotted quantitative data into a clean Markdown table with column headers."} ] }] text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)image_inputs, _ = process_vision_info(messages)inputs = processor(text=[text], images=image_inputs, padding=True, return_tensors="pt").to("cuda") with torch.inference_mode(): generated_ids = model.generate(**inputs, max_new_tokens=1024, temperature=0.0) generated_ids_trimmed = [ out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids) ] output_text = processor.batch_decode( generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False )[0] print("--- Extracted Markdown Table ---")print(output_text)