GLM-5.3 is live. Run Z.ai's latest model on Friendli Model APIs. Try it today ➜
blind-assist
Available on FriendliAI
Run this model inference on single tenant GPU with unmatched speed and reliability at scale.
Model Details
Model Provider
blind-assist
Model Tree
OpenGVLab/InternVL3-1B
Input Modalities
Output Modalities
Supported Functionality
| Attribute | Value |
|---|---|
| Base Model | OpenGVLab/InternVL3-1B |
| Method | LoRA (Low-Rank Adaptation) |
| LoRA Rank | 128 |
| Target Modules | down_proj, gate_proj, k_proj, o_proj, q_proj, up_proj, v_proj |
| Task | Navigation hazard detection |
python
import torchfrom transformers import AutoModel, AutoTokenizerfrom huggingface_hub import hf_hub_downloadfrom safetensors.torch import load_file # Load base modelmodel = AutoModel.from_pretrained( "OpenGVLab/InternVL3-1B", torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto")tokenizer = AutoTokenizer.from_pretrained("OpenGVLab/InternVL3-1B", trust_remote_code=True) # Download adapter weightsadapter_path = hf_hub_download("blind-assist/internvl3-1b-walk-lora-Epoch_1.85-8500-without_ES_v1", "adapter_model.safetensors")adapter_weights = load_file(adapter_path) # Merge LoRA weightsmodel_state = model.state_dict()scaling = 1.0 # lora_alpha / lora_r = 128 / 128 for key in adapter_weights: if '.lora_A.' in key: lora_b_key = key.replace('.lora_A.', '.lora_B.') if lora_b_key in adapter_weights: model_key = key.replace('.lora_A.', '.').replace('base_model.model.', '') if model_key in model_state: lora_a = adapter_weights[key].float().to(model_state[model_key].device) lora_b = adapter_weights[lora_b_key].float().to(model_state[model_key].device) delta = torch.matmul(lora_b, lora_a) * scaling model_state[model_key] = model_state[model_key].float() + delta model_state[model_key] = model_state[model_key].to(torch.bfloat16) elif '.lora_B.' not in key: # Load other fine-tuned weights model_key = key.replace('base_model.model.', '') if model_key in model_state and model_state[model_key].shape == adapter_weights[key].shape: model_state[model_key] = adapter_weights[key].to(model_state[model_key].device) model.load_state_dict(model_state)model.eval() # Inferenceprompt = "Given the visual input from the user's forward perspective, generate exactly one short sentence to guide a visually impaired user by identifying critical obstacles or landmarks, describing their locations using clock directions relative to the user (12 o'clock is straight ahead), including relevant details such as size, material, or distance, and giving one clear action, while prioritizing immediate safety and avoiding any extra explanation." response = model.chat( tokenizer=tokenizer, pixel_values=your_image_tensor, # Preprocessed image question=prompt, generation_config=dict(max_new_tokens=256, do_sample=False))print(response)bibtex
@misc{blindassist2024walkvlm, title={WalkVLM: Fine-tuned Vision-Language Model for Blind Navigation}, author={Blind-Assist Team}, year={2024}, url={https://huggingface.co/blind-assist/internvl3-1b-walk-lora-Epoch_1.85-8500-without_ES_v1}}import torchfrom transformers import AutoModel, AutoTokenizerfrom huggingface_hub import hf_hub_downloadfrom safetensors.torch import load_file # Load base modelmodel = AutoModel.from_pretrained( "OpenGVLab/InternVL3-1B", torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto")tokenizer = AutoTokenizer.from_pretrained("OpenGVLab/InternVL3-1B", trust_remote_code=True) # Download adapter weightsadapter_path = hf_hub_download("blind-assist/internvl3-1b-walk-lora-Epoch_1.85-8500-without_ES_v1", "adapter_model.safetensors")adapter_weights = load_file(adapter_path) # Merge LoRA weightsmodel_state = model.state_dict()scaling = 1.0 # lora_alpha / lora_r = 128 / 128 for key in adapter_weights: if '.lora_A.' in key: lora_b_key = key.replace('.lora_A.', '.lora_B.') if lora_b_key in adapter_weights: model_key = key.replace('.lora_A.', '.').replace('base_model.model.', '') if model_key in model_state: lora_a = adapter_weights[key].float().to(model_state[model_key].device) lora_b = adapter_weights[lora_b_key].float().to(model_state[model_key].device) delta = torch.matmul(lora_b, lora_a) * scaling model_state[model_key] = model_state[model_key].float() + delta model_state[model_key] = model_state[model_key].to(torch.bfloat16) elif '.lora_B.' not in key: # Load other fine-tuned weights model_key = key.replace('base_model.model.', '') if model_key in model_state and model_state[model_key].shape == adapter_weights[key].shape: model_state[model_key] = adapter_weights[key].to(model_state[model_key].device) model.load_state_dict(model_state)model.eval() # Inferenceprompt = "Given the visual input from the user's forward perspective, generate exactly one short sentence to guide a visually impaired user by identifying critical obstacles or landmarks, describing their locations using clock directions relative to the user (12 o'clock is straight ahead), including relevant details such as size, material, or distance, and giving one clear action, while prioritizing immediate safety and avoiding any extra explanation." response = model.chat( tokenizer=tokenizer, pixel_values=your_image_tensor, # Preprocessed image question=prompt, generation_config=dict(max_new_tokens=256, do_sample=False))print(response)@misc{blindassist2024walkvlm, title={WalkVLM: Fine-tuned Vision-Language Model for Blind Navigation}, author={Blind-Assist Team}, year={2024}, url={https://huggingface.co/blind-assist/internvl3-1b-walk-lora-Epoch_1.85-8500-without_ES_v1}}