qwen2.5-7b-pii-financial API & Inference Endpoint

This model is deployed as a serverless web endpoint on Modal using an A10G GPU container instance (serve web function), making it easy to query via standard HTTP requests.

python
import os
import requests

url = "[https://durganani60--pii-redaction-qwen-serve.modal.run/v1/chat/completions](https://durganani60--pii-redaction-qwen-serve.modal.run/v1/chat/completions)"

# Retrieve your API key from environment variables or provide it directly
api_key = os.environ.get("MODAL_API_KEY", "YOUR_MODAL_API_KEY")

headers = {
    "Content-Type": "application/json",
    "Authorization": f"Bearer {api_key}",
}

system_prompt = (
    "You are a PII redaction engine for financial documents. Find every span "
    "of personally identifiable information in the document: names, companies, "
    "dates, street addresses, account or routing numbers, emails, phone "
    "numbers, and any other identifier tied to a person or organization. "
    'Respond with a single JSON object with two keys: "redacted_text", the '
    "full document with each PII span replaced by its type in uppercase "
    'square brackets such as [NAME] or [EMAIL], and "entities", a list of '
    '{"type": ..., "value": ...} objects, one per span in order of '
    "appearance. Respond with only the JSON object."
)

payload = {
    "model": "qwen3-8b-pii",
    "messages": [
        {"role": "system", "content": system_prompt},
        {
            "role": "user",
            "content": "John Doe transferred $5,000 from account 123456789 on March 14, 2024.",
        },
    ],
    "temperature": 0,
}

response = requests.post(url, headers=headers, json=payload)

if response.status_code == 200:
    print(response.json()["choices"][0]["message"]["content"])
else:
    print(f"Error {response.status_code}: {response.text}")

# Verified Response Output

{
  "redacted_text": "[NAME] transferred $5,000 from account [ACCOUNT_NUMBER] on [DATE].",
  "entities": [
    {"type": "name", "value": "John Doe"},
    {"type": "account_number", "value": "123456789"},
    {"type": "date", "value": "March 14, 2024"}
  ]
}

2. Quick Start (Google Colab Inference with 4-bit Quantization)

You can easily test and run this adapter in a free or low-cost cloud GPU environment (such as an NVIDIA T4) using 4-bit bitsandbytes quantization to manage VRAM limits efficiently.

Access the live reference notebook here: https://colab.research.google.com/drive/1DJqmlAdEt_SegoYLCDdGSBW9u6ieMIfF#scrollTo=S4GptehjYtFo

python
!pip install --upgrade torchao
!pip install -q bitsandbytes accelerate

import torch
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

# Define your base model and adapter repository
base_model_id = "Qwen/Qwen2.5-7B-Instruct"
adapter_id = "durganani60/qwen2.5-7b-pii-financial"

# Configure 4-bit quantization to fit the model within T4 VRAM limits
quantization_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_use_double_quant=True
)

print("Loading base model in 4-bit...")
tokenizer = AutoTokenizer.from_pretrained(base_model_id)

base_model = AutoModelForCausalLM.from_pretrained(
    base_model_id, 
    quantization_config=quantization_config, 
    device_map="auto"
)

# Attach your fine-tuned PII adapter to the base model
print("Loading fine-tuned PII adapter...")
model = PeftModel.from_pretrained(base_model, adapter_id)

# Define system prompt and test input
system_prompt = (
    "You are a PII redaction engine for financial documents. Find every span "
    "of personally identifiable information in the document: names, companies, "
    "dates, street addresses, account or routing numbers, emails, phone "
    "numbers, and any other identifier tied to a person or organization. "
    'Respond with a single JSON object with two keys: "redacted_text", the '
    "full document with each PII span replaced by its type in uppercase "
    'square brackets such as [NAME] or [EMAIL], and "entities", a list of '
    '{"type": ..., "value": ...} objects, one per span in order of '
    "appearance. Respond with only the JSON object."
)

user_input = "John Doe transferred $5,000 from account 123456789 on March 14, 2024."

messages = [
    {"role": "system", "content": system_prompt},
    {"role": "user", "content": user_input},
]

# Tokenize and generate output
text = tokenizer.apply_chat_template(
    messages, tokenize=False, add_generation_prompt=True
)
inputs = tokenizer(text, return_tensors="pt").to("cuda")

print("Generating prediction...")
outputs = model.generate(
    **inputs, max_new_tokens=512, temperature=0.0, do_sample=False
)
response = tokenizer.decode(
    outputs[0][inputs["input_ids"].shape[1] :], skip_special_tokens=True
)

print("\n--- Model Output ---")
print(response)

# Verified Response Output

{
    "redacted_text": "[NAME] transferred $5,000 from account [ACCOUNT_NUMBER] on [DATE].", 
    "entities": 
    [
        {
            "type": "name", "value": "John Doe"
        }, 
        {
            "type": "account_number", "value": "123456789"
        }, 
        {
            "type": "date", "value": "March 14, 2024"
        }
    ]
}

python

import os
import requests

url = "[https://durganani60--pii-redaction-qwen-serve.modal.run/v1/chat/completions](https://durganani60--pii-redaction-qwen-serve.modal.run/v1/chat/completions)"

# Retrieve your API key from environment variables or provide it directly
api_key = os.environ.get("MODAL_API_KEY", "YOUR_MODAL_API_KEY")

headers = {
    "Content-Type": "application/json",
    "Authorization": f"Bearer {api_key}",
}

system_prompt = (
    "You are a PII redaction engine for financial documents. Find every span "
    "of personally identifiable information in the document: names, companies, "
    "dates, street addresses, account or routing numbers, emails, phone "
    "numbers, and any other identifier tied to a person or organization. "
    'Respond with a single JSON object with two keys: "redacted_text", the '
    "full document with each PII span replaced by its type in uppercase "
    'square brackets such as [NAME] or [EMAIL], and "entities", a list of '
    '{"type": ..., "value": ...} objects, one per span in order of '
    "appearance. Respond with only the JSON object."
)

payload = {
    "model": "qwen3-8b-pii",
    "messages": [
        {"role": "system", "content": system_prompt},
        {
            "role": "user",
            "content": "John Doe transferred $5,000 from account 123456789 on March 14, 2024.",
        },
    ],
    "temperature": 0,
}

response = requests.post(url, headers=headers, json=payload)

if response.status_code == 200:
    print(response.json()["choices"][0]["message"]["content"])
else:
    print(f"Error {response.status_code}: {response.text}")

# Verified Response Output

{
  "redacted_text": "[NAME] transferred $5,000 from account [ACCOUNT_NUMBER] on [DATE].",
  "entities": [
    {"type": "name", "value": "John Doe"},
    {"type": "account_number", "value": "123456789"},
    {"type": "date", "value": "March 14, 2024"}
  ]
}

2. Quick Start (Google Colab Inference with 4-bit Quantization)

You can easily test and run this adapter in a free or low-cost cloud GPU environment (such as an NVIDIA T4) using 4-bit bitsandbytes quantization to manage VRAM limits efficiently.

python

!pip install --upgrade torchao
!pip install -q bitsandbytes accelerate

import torch
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

# Define your base model and adapter repository
base_model_id = "Qwen/Qwen2.5-7B-Instruct"
adapter_id = "durganani60/qwen2.5-7b-pii-financial"

# Configure 4-bit quantization to fit the model within T4 VRAM limits
quantization_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_use_double_quant=True
)

print("Loading base model in 4-bit...")
tokenizer = AutoTokenizer.from_pretrained(base_model_id)

base_model = AutoModelForCausalLM.from_pretrained(
    base_model_id, 
    quantization_config=quantization_config, 
    device_map="auto"
)

# Attach your fine-tuned PII adapter to the base model
print("Loading fine-tuned PII adapter...")
model = PeftModel.from_pretrained(base_model, adapter_id)

# Define system prompt and test input
system_prompt = (
    "You are a PII redaction engine for financial documents. Find every span "
    "of personally identifiable information in the document: names, companies, "
    "dates, street addresses, account or routing numbers, emails, phone "
    "numbers, and any other identifier tied to a person or organization. "
    'Respond with a single JSON object with two keys: "redacted_text", the '
    "full document with each PII span replaced by its type in uppercase "
    'square brackets such as [NAME] or [EMAIL], and "entities", a list of '
    '{"type": ..., "value": ...} objects, one per span in order of '
    "appearance. Respond with only the JSON object."
)

user_input = "John Doe transferred $5,000 from account 123456789 on March 14, 2024."

messages = [
    {"role": "system", "content": system_prompt},
    {"role": "user", "content": user_input},
]

# Tokenize and generate output
text = tokenizer.apply_chat_template(
    messages, tokenize=False, add_generation_prompt=True
)
inputs = tokenizer(text, return_tensors="pt").to("cuda")

print("Generating prediction...")
outputs = model.generate(
    **inputs, max_new_tokens=512, temperature=0.0, do_sample=False
)
response = tokenizer.decode(
    outputs[0][inputs["input_ids"].shape[1] :], skip_special_tokens=True
)

print("\n--- Model Output ---")
print(response)

# Verified Response Output

{
    "redacted_text": "[NAME] transferred $5,000 from account [ACCOUNT_NUMBER] on [DATE].", 
    "entities": 
    [
        {
            "type": "name", "value": "John Doe"
        }, 
        {
            "type": "account_number", "value": "123456789"
        }, 
        {
            "type": "date", "value": "March 14, 2024"
        }
    ]
}

qwen2.5-7b-pii-financial

README

2. Quick Start (Google Colab Inference with 4-bit Quantization)

Explore FriendliAI today

README

2. Quick Start (Google Colab Inference with 4-bit Quantization)

qwen2.5-7b-pii-financial

README

1. Production Deployment (Modal Server Endpoint)

2. Quick Start (Google Colab Inference with 4-bit Quantization)

Explore FriendliAI today

README

1. Production Deployment (Modal Server Endpoint)

2. Quick Start (Google Colab Inference with 4-bit Quantization)