huihui-ai/DeepSeek-V4-Flash-BF16

This model converted from deepseek-ai/DeepSeek-V4-Flash.

Note This is not the ablated model.

If you don't have enough GPU memory, we recommend testing it using CPU memory.

Transformers

Use the latest version of transformers.

markdown
Copy code
pip install transformers -U

markdown
Copy code
from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer
import torch
import os
import signal
import random
import numpy as np
import time
from collections import Counter
import warnings
from transformers import logging
warnings.filterwarnings("ignore", message="Unrecognized keys in `rope_parameters`")
logging.set_verbosity_error()

cpu_count = os.cpu_count()
print(f"Number of CPU cores in the system: {cpu_count}")
half_cpu_count = cpu_count // 2
os.environ["MKL_NUM_THREADS"] = str(half_cpu_count)
os.environ["OMP_NUM_THREADS"] = str(half_cpu_count)
torch.set_num_threads(half_cpu_count)

print(f"PyTorch threads: {torch.get_num_threads()}")
print(f"MKL threads: {os.getenv('MKL_NUM_THREADS')}")
print(f"OMP threads: {os.getenv('OMP_NUM_THREADS')}")

torch.set_default_dtype(torch.bfloat16)

# Load the model and tokenizer
MODEL_ID = "deepseek-ai/DeepSeek-V4-Flash-BF16"

print(f"Load Model {MODEL_ID} ... ")
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    torch_dtype=torch.bfloat16,
    device_map="cpu",
    low_cpu_mem_usage=True,
    trust_remote_code=True,

    offload_folder="./offload",
)

model = model.to(torch.bfloat16)

tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
#if tokenizer.chat_template is None:
#    tokenizer.chat_template = """<｜begin▁of▁sentence｜>You are a helpful assistant.{% for message in messages %}{% if message['role'] == 'user' %}<｜User｜>{{ message['content'] }}<｜Assistant｜>{% elif message['role'] == 'assistant' %}{{ message['content'] }}<｜end▁of▁sentence｜>{% endif %}{% endfor %}"""

class CustomTextStreamer(TextStreamer):
    def __init__(self, tokenizer, skip_prompt=True, skip_special_tokens=True):
        super().__init__(tokenizer, skip_prompt=skip_prompt, skip_special_tokens=skip_special_tokens)
        self.generated_text = ""
        self.stop_flag = False
        self.init_time = time.time()  # Record initialization time
        self.end_time = None  # To store end time
        self.first_token_time = None  # To store first token generation time
        self.think_tokens_count = 0  # To track total think tokens
        self.token_count = 0  # To track total tokens

    def on_finalized_text(self, text: str, stream_end: bool = False):
        if self.first_token_time is None and text.strip():  # Set first token time on first non-empty text
            self.first_token_time = time.time()
        self.generated_text += text
        # Count tokens in the generated text
        tokens = self.tokenizer.encode(text, add_special_tokens=False)
        self.token_count += len(tokens)
        if self.think_tokens_count == 0 and "</think>" in self.generated_text:
              self.think_tokens_count = self.token_count
        print(text, end="", flush=True)
        if stream_end:
            self.end_time = time.time()  # Record end time when streaming ends
        if self.stop_flag:
            raise StopIteration

    def stop_generation(self):
        self.stop_flag = True
        self.end_time = time.time()  # Record end time when generation is stopped

    def get_metrics(self):
        """Returns initialization time, first token time, first token latency, end time, total time, total tokens, and tokens per second."""
        if self.end_time is None:
            self.end_time = time.time()  # Set end time if not already set
        total_time = self.end_time - self.init_time  # Total time from init to end
        tokens_per_second = self.token_count / total_time if total_time > 0 else 0
        first_token_latency = (self.first_token_time - self.init_time) if self.first_token_time is not None else None
        metrics = {
            "init_time": self.init_time,
            "first_token_time": self.first_token_time,
            "first_token_latency": first_token_latency,
            "end_time": self.end_time,
            "total_time": total_time,  # Total time in seconds
            "think_tokens_count": self.think_tokens_count,
            "total_tokens": self.token_count,
            "tokens_per_second": tokens_per_second
        }
        return metrics

def generate_stream(model, tokenizer, messages, thinking_mode, skip_prompt, skip_special_tokens, max_new_tokens):
    if thinking_mode:
        formatted_prompt = tokenizer.apply_chat_template(
            messages,
            tokenize=False,
            thinking_mode="thinking",
            add_generation_prompt=True,
        )
    else:
        formatted_prompt = tokenizer.apply_chat_template(
            messages,
            tokenize=False,
            thinking_mode="chat",
            add_generation_prompt=True,
        )

    print(f"formatted_prompt={formatted_prompt}\n")

    toks = tokenizer(
        [formatted_prompt],
        return_tensors="pt",
        return_token_type_ids=False,
    ).to(model.device)

    streamer = CustomTextStreamer(tokenizer, skip_prompt=skip_prompt, skip_special_tokens=skip_special_tokens)

    def signal_handler(sig, frame):
        streamer.stop_generation()
        print("\n[Generation stopped by user with Ctrl+C]")

    signal.signal(signal.SIGINT, signal_handler)

    print("Response: ", end="", flush=True)
    try:
        generated_ids = model.generate(
            **toks,
            max_new_tokens=max_new_tokens,
            pad_token_id=tokenizer.eos_token_id,
            streamer=streamer,
         )
        del generated_ids
    except StopIteration:
        print("\n[Stopped by user]")

    del toks
    torch.cuda.empty_cache()
    signal.signal(signal.SIGINT, signal.SIG_DFL)

    return streamer.generated_text, streamer.stop_flag, streamer.get_metrics()

init_messages = [{"role": "system", "content": "You are a helpful assistant."}]
messages = init_messages.copy()

skip_prompt=False
skip_special_tokens=False
thinking_mode=True

while True:
    print(f"skip_prompt: {skip_prompt}")
    print(f"skip_special_tokens: {skip_special_tokens}")
    print(f"thinking_mode: {thinking_mode}")

    user_input = input("User: ").strip()
    if user_input.lower() == "/exit":
        print("Exiting chat.")
        break
    if user_input.lower() == "/clear":
        messages = init_messages.copy()
        print("Chat history cleared. Starting a new conversation.")
        continue
    if user_input.lower() == "/skip_prompt":
        skip_prompt = not skip_prompt
        continue
    if user_input.lower() == "/skip_special_tokens":
        skip_special_tokens = not skip_special_tokens
        continue
    if user_input.lower() == "/thinking_mode":
        thinking_mode = not thinking_mode
        continue
    if not user_input:
        print("Input cannot be empty. Please enter something.")
        continue


    messages.append({"role": "user", "content": user_input})
    activated_experts = []
    response, stop_flag, metrics = generate_stream(model, tokenizer, messages, thinking_mode, skip_prompt, skip_special_tokens, 40960)
    print("\n\nMetrics:")
    for key, value in metrics.items():
        print(f"  {key}: {value}")

    print("", flush=True)
    if stop_flag:
        continue
    messages.append({"role": "assistant", "content": response})

Donation

You can follow x.com/support_huihui to get the latest model information from huihui.ai.

Your donation helps us continue our further development and improvement, a cup of coffee can do it.

• bitcoin:

markdown
Copy code
bc1qqnkhuchxw0zqjh2ku3lu4hq45hc6gy84uk70ge

• Support our work on Ko-fi!