deepseek-v4-bf16-tiny-random API & Inference Endpoint

This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from deepseek-ai/DeepSeek-V4-Pro.

Note:

This model is in bfloat16, not quantized to fp8/fp4.
Does not contain MTP layer.
Chat template from this PR.

Table with columns: File path, Size
File path	Size
model.safetensors	30.5MB

Example usage:

python
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

model_id = "yujiepan/deepseek-v4-bf16-tiny-random"
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device}")
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)

# Note: `.from_pretrained()` automatically casts some layers to float32, based on
# `_keep_in_fp32_modules_strict`, causing fp32 x bf16 cases.
# Seems like a bug. Here we force bf16 dtype as a workaround.
# Tested on transformers 5.13.0.
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    dtype=torch.bfloat16,
    trust_remote_code=True,
).to(device).to(torch.bfloat16)
model.eval()

inputs = tokenizer.apply_chat_template(
    [
        {"role": "user", "content": f"Hello! Counting 1 to 200: {list(range(1, 201))}"}
    ],
    tools=[
        {
            "name": "count",
            "description": "Count numbers",
            "parameters": {
                    "type": "object",
                    "properties": {
                        "numbers": {"type": "array", "items": {"type": "integer"}},
                    },
                "required": ["numbers"],
            },
        },
    ],
    return_tensors="pt",
    add_generation_prompt=True,
    thinking_mode="thinking",
    reasoning_effort="max",
).to(device)
outputs = model.generate(**inputs, max_new_tokens=32, max_length=1024)
print(tokenizer.decode(outputs[0], skip_special_tokens=False))

Codes to create this repo:

python
import heapq
import json

import jinja2
import torch
from huggingface_hub import file_exists, hf_hub_download
from transformers import (
    AutoConfig,
    AutoModelForCausalLM,
    AutoTokenizer,
    DeepseekV4Config,
    GenerationConfig,
    set_seed,
)

source_model_id = "deepseek-ai/DeepSeek-V4-Pro"
save_folder = "/tmp/yujiepan/deepseek-v4-bf16-tiny-random"  # pyright: ignore[reportUnusedExpression]

tokenizer = AutoTokenizer.from_pretrained(
    source_model_id, trust_remote_code=True,
)
if file_exists(filename="chat_template.jinja", repo_id=source_model_id, repo_type='model', revision="refs/pr/146"):
    with open(hf_hub_download(
        source_model_id,
        filename="chat_template.jinja",
        repo_type='model',
        revision="refs/pr/146",
    ), 'r', encoding='utf-8') as f:
        tokenizer.chat_template = f.read()
tokenizer.save_pretrained(save_folder)

with open(hf_hub_download(
    source_model_id,
    filename='config.json',
    repo_type='model',
), 'r', encoding='utf-8') as f:
    config_json = json.load(f)

# Modify config_json here before loading.
config_json.update({
    "head_dim": 128,
    "hidden_size": 8,
    "index_head_dim": 128,
    "index_n_heads": 4,
    "moe_intermediate_size": 32,
    "n_routed_experts": 256,
    # "n_shared_experts": 1,
    "num_attention_heads": 8,
    "num_hidden_layers": 7,
    "num_experts_per_tok": 6,
    "num_hash_layers": 3,
    "num_key_value_heads": 1,
    "q_lora_rank": 128,
    "o_lora_rank": 128,
    "o_groups": 8,
    "layer_types": [
        "sliding_attention",
        "sliding_attention",
        "compressed_sparse_attention",
        "heavily_compressed_attention",
        "compressed_sparse_attention",
        "heavily_compressed_attention",
        "compressed_sparse_attention"
    ],
    "compress_ratios": [0, 0, 4, 128, 4, 128, 4, 0],
})
del config_json['quantization_config']
del config_json['expert_dtype']

with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f:
    json.dump(config_json, f, indent=2)

config = AutoConfig.from_pretrained(
    save_folder,
    trust_remote_code=True,
)
assert isinstance(config, DeepseekV4Config)
model = AutoModelForCausalLM.from_config(
    config,
    dtype=torch.bfloat16,
    trust_remote_code=True,
)
if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'):
    model.generation_config = GenerationConfig.from_pretrained(
        source_model_id, trust_remote_code=True,
    )
set_seed(42)
max_width = max(len(name) for name in model.state_dict().keys())
name2size = {name: tensor.numel() * tensor.element_size() for name, tensor in model.state_dict().items()}

print("Top 10 tensor sizes:")
for name, size in heapq.nlargest(10, name2size.items(), key=lambda x: x[1]):
    print(
        name.ljust(max_width),
        f'{size / 1024**2:.2f}MB',
        model.state_dict()[name].shape,
    )

print("Initializing parameters:")
with torch.no_grad():
    for name, p in sorted(model.named_parameters()):
        torch.nn.init.normal_(p, 0, 0.3)
        print(
            name.ljust(max_width),
            f'{name2size[name] / 1024**2:.2f}MB',
            p.shape, p.dtype,
        )
    for name, b in model.named_buffers():
        print(
            name.ljust(max_width),
            f'{b.numel() * b.element_size() / 1024**2:.2f}MB',
            b.shape, b.dtype,
        )
        if name.endswith("e_score_correction_bias"):
            torch.nn.init.normal_(b, 0, 0.3)
        elif name.endswith("tid2eid"):
            b.copy_(torch.rand(
                b.shape[0], config.n_routed_experts, device=b.device,
            ).topk(b.shape[1], dim=-1).indices.to(b.dtype))
        elif name.endswith("inv_freq"):
            pass
        else:
            raise ValueError(f"Unknown buffer: {name}")

model.save_pretrained(save_folder)
with open(f"{save_folder}/config.json", "r", encoding="utf-8") as f:
    saved_config_json = json.load(f)

# vLLM reads old config.json, so we need to modify it to make it compatible.
saved_config_json.pop("mlp_layer_types", None)
saved_config_json["num_hash_layers"] = config_json["num_hash_layers"]
saved_config_json["compress_ratios"] = [0, 0, 4, 128, 4, 128, 4, 0]
rope_parameters = saved_config_json.pop("rope_parameters", None)
saved_config_json["rope_scaling"] = {
    "beta_fast": 32,
    "beta_slow": 1,
    "factor": 16,
    "original_max_position_embeddings": 65536,
    "type": "yarn"
}
saved_config_json["rope_theta"] = 10000
saved_config_json["compress_rope_theta"] = 160000
with open(f"{save_folder}/config.json", "w", encoding="utf-8") as f:
    json.dump(saved_config_json, f, indent=2)

Printing the model:

text
DeepseekV4ForCausalLM(
  (model): DeepseekV4Model(
    (embed_tokens): Embedding(129280, 8)
    (layers): ModuleList(
      (0-1): 2 x DeepseekV4DecoderLayer(
        (self_attn): DeepseekV4Attention(
          (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
          (q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
          (q_b_norm): DeepseekV4UnweightedRMSNorm()
          (kv_proj): Linear(in_features=8, out_features=128, bias=False)
          (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
          (o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
        )
        (mlp): DeepseekV4SparseMoeBlock(
          (gate): DeepseekV4HashRouter(
            (score_fn): SqrtSoftplusActivation()
          )
          (experts): DeepseekV4Experts(
            (act_fn): SiLUActivation()
          )
          (shared_experts): DeepseekV4MLP(
            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
            (up_proj): Linear(in_features=8, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=8, bias=False)
            (act_fn): SiLUActivation()
          )
        )
        (input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (attn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
        (ffn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
      )
      (2): DeepseekV4DecoderLayer(
        (self_attn): DeepseekV4Attention(
          (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
          (q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
          (q_b_norm): DeepseekV4UnweightedRMSNorm()
          (kv_proj): Linear(in_features=8, out_features=128, bias=False)
          (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
          (o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
          (compressor): DeepseekV4CSACompressor(
            (kv_proj): Linear(in_features=8, out_features=256, bias=False)
            (gate_proj): Linear(in_features=8, out_features=256, bias=False)
            (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
            (rotary_emb): DeepseekV4RotaryEmbedding()
            (indexer): DeepseekV4Indexer(
              (kv_proj): Linear(in_features=8, out_features=256, bias=False)
              (gate_proj): Linear(in_features=8, out_features=256, bias=False)
              (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
              (q_b_proj): Linear(in_features=128, out_features=512, bias=False)
              (rotary_emb): DeepseekV4RotaryEmbedding()
              (scorer): DeepseekV4IndexerScorer(
                (weights_proj): Linear(in_features=8, out_features=4, bias=False)
              )
            )
          )
        )
        (mlp): DeepseekV4SparseMoeBlock(
          (gate): DeepseekV4HashRouter(
            (score_fn): SqrtSoftplusActivation()
          )
          (experts): DeepseekV4Experts(
            (act_fn): SiLUActivation()
          )
          (shared_experts): DeepseekV4MLP(
            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
            (up_proj): Linear(in_features=8, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=8, bias=False)
            (act_fn): SiLUActivation()
          )
        )
        (input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (attn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
        (ffn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
      )
      (3): DeepseekV4DecoderLayer(
        (self_attn): DeepseekV4Attention(
          (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
          (q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
          (q_b_norm): DeepseekV4UnweightedRMSNorm()
          (kv_proj): Linear(in_features=8, out_features=128, bias=False)
          (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
          (o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
          (compressor): DeepseekV4HCACompressor(
            (kv_proj): Linear(in_features=8, out_features=128, bias=False)
            (gate_proj): Linear(in_features=8, out_features=128, bias=False)
            (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
            (rotary_emb): DeepseekV4RotaryEmbedding()
          )
        )
        (mlp): DeepseekV4SparseMoeBlock(
          (gate): DeepseekV4TopKRouter(
            (score_fn): SqrtSoftplusActivation()
          )
          (experts): DeepseekV4Experts(
            (act_fn): SiLUActivation()
          )
          (shared_experts): DeepseekV4MLP(
            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
            (up_proj): Linear(in_features=8, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=8, bias=False)
            (act_fn): SiLUActivation()
          )
        )
        (input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (attn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
        (ffn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
      )
      (4): DeepseekV4DecoderLayer(
        (self_attn): DeepseekV4Attention(
          (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
          (q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
          (q_b_norm): DeepseekV4UnweightedRMSNorm()
          (kv_proj): Linear(in_features=8, out_features=128, bias=False)
          (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
          (o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
          (compressor): DeepseekV4CSACompressor(
            (kv_proj): Linear(in_features=8, out_features=256, bias=False)
            (gate_proj): Linear(in_features=8, out_features=256, bias=False)
            (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
            (rotary_emb): DeepseekV4RotaryEmbedding()
            (indexer): DeepseekV4Indexer(
              (kv_proj): Linear(in_features=8, out_features=256, bias=False)
              (gate_proj): Linear(in_features=8, out_features=256, bias=False)
              (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
              (q_b_proj): Linear(in_features=128, out_features=512, bias=False)
              (rotary_emb): DeepseekV4RotaryEmbedding()
              (scorer): DeepseekV4IndexerScorer(
                (weights_proj): Linear(in_features=8, out_features=4, bias=False)
              )
            )
          )
        )
        (mlp): DeepseekV4SparseMoeBlock(
          (gate): DeepseekV4TopKRouter(
            (score_fn): SqrtSoftplusActivation()
          )
          (experts): DeepseekV4Experts(
            (act_fn): SiLUActivation()
          )
          (shared_experts): DeepseekV4MLP(
            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
            (up_proj): Linear(in_features=8, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=8, bias=False)
            (act_fn): SiLUActivation()
          )
        )
        (input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (attn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
        (ffn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
      )
      (5): DeepseekV4DecoderLayer(
        (self_attn): DeepseekV4Attention(
          (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
          (q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
          (q_b_norm): DeepseekV4UnweightedRMSNorm()
          (kv_proj): Linear(in_features=8, out_features=128, bias=False)
          (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
          (o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
          (compressor): DeepseekV4HCACompressor(
            (kv_proj): Linear(in_features=8, out_features=128, bias=False)
            (gate_proj): Linear(in_features=8, out_features=128, bias=False)
            (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
            (rotary_emb): DeepseekV4RotaryEmbedding()
          )
        )
        (mlp): DeepseekV4SparseMoeBlock(
          (gate): DeepseekV4TopKRouter(
            (score_fn): SqrtSoftplusActivation()
          )
          (experts): DeepseekV4Experts(
            (act_fn): SiLUActivation()
          )
          (shared_experts): DeepseekV4MLP(
            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
            (up_proj): Linear(in_features=8, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=8, bias=False)
            (act_fn): SiLUActivation()
          )
        )
        (input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (attn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
        (ffn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
      )
      (6): DeepseekV4DecoderLayer(
        (self_attn): DeepseekV4Attention(
          (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
          (q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
          (q_b_norm): DeepseekV4UnweightedRMSNorm()
          (kv_proj): Linear(in_features=8, out_features=128, bias=False)
          (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
          (o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
          (compressor): DeepseekV4CSACompressor(
            (kv_proj): Linear(in_features=8, out_features=256, bias=False)
            (gate_proj): Linear(in_features=8, out_features=256, bias=False)
            (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
            (rotary_emb): DeepseekV4RotaryEmbedding()
            (indexer): DeepseekV4Indexer(
              (kv_proj): Linear(in_features=8, out_features=256, bias=False)
              (gate_proj): Linear(in_features=8, out_features=256, bias=False)
              (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
              (q_b_proj): Linear(in_features=128, out_features=512, bias=False)
              (rotary_emb): DeepseekV4RotaryEmbedding()
              (scorer): DeepseekV4IndexerScorer(
                (weights_proj): Linear(in_features=8, out_features=4, bias=False)
              )
            )
          )
        )
        (mlp): DeepseekV4SparseMoeBlock(
          (gate): DeepseekV4TopKRouter(
            (score_fn): SqrtSoftplusActivation()
          )
          (experts): DeepseekV4Experts(
            (act_fn): SiLUActivation()
          )
          (shared_experts): DeepseekV4MLP(
            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
            (up_proj): Linear(in_features=8, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=8, bias=False)
            (act_fn): SiLUActivation()
          )
        )
        (input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (attn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
        (ffn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
      )
    )
    (norm): DeepseekV4RMSNorm((8,), eps=1e-06)
    (rotary_emb): DeepseekV4RotaryEmbedding()
    (hc_head): DeepseekV4HyperHead(
      (input_norm): DeepseekV4UnweightedRMSNorm()
    )
  )
  (lm_head): Linear(in_features=8, out_features=129280, bias=False)
)

Test environment:

torch: 2.11.0+cu128
transformers: 5.13.0

Change log:

2026-07-06: Initial version.

File path

Size

model.safetensors

30.5MB

python

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

model_id = "yujiepan/deepseek-v4-bf16-tiny-random"
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device}")
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)

# Note: `.from_pretrained()` automatically casts some layers to float32, based on
# `_keep_in_fp32_modules_strict`, causing fp32 x bf16 cases.
# Seems like a bug. Here we force bf16 dtype as a workaround.
# Tested on transformers 5.13.0.
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    dtype=torch.bfloat16,
    trust_remote_code=True,
).to(device).to(torch.bfloat16)
model.eval()

inputs = tokenizer.apply_chat_template(
    [
        {"role": "user", "content": f"Hello! Counting 1 to 200: {list(range(1, 201))}"}
    ],
    tools=[
        {
            "name": "count",
            "description": "Count numbers",
            "parameters": {
                    "type": "object",
                    "properties": {
                        "numbers": {"type": "array", "items": {"type": "integer"}},
                    },
                "required": ["numbers"],
            },
        },
    ],
    return_tensors="pt",
    add_generation_prompt=True,
    thinking_mode="thinking",
    reasoning_effort="max",
).to(device)
outputs = model.generate(**inputs, max_new_tokens=32, max_length=1024)
print(tokenizer.decode(outputs[0], skip_special_tokens=False))

python

import heapq
import json

import jinja2
import torch
from huggingface_hub import file_exists, hf_hub_download
from transformers import (
    AutoConfig,
    AutoModelForCausalLM,
    AutoTokenizer,
    DeepseekV4Config,
    GenerationConfig,
    set_seed,
)

source_model_id = "deepseek-ai/DeepSeek-V4-Pro"
save_folder = "/tmp/yujiepan/deepseek-v4-bf16-tiny-random"  # pyright: ignore[reportUnusedExpression]

tokenizer = AutoTokenizer.from_pretrained(
    source_model_id, trust_remote_code=True,
)
if file_exists(filename="chat_template.jinja", repo_id=source_model_id, repo_type='model', revision="refs/pr/146"):
    with open(hf_hub_download(
        source_model_id,
        filename="chat_template.jinja",
        repo_type='model',
        revision="refs/pr/146",
    ), 'r', encoding='utf-8') as f:
        tokenizer.chat_template = f.read()
tokenizer.save_pretrained(save_folder)

with open(hf_hub_download(
    source_model_id,
    filename='config.json',
    repo_type='model',
), 'r', encoding='utf-8') as f:
    config_json = json.load(f)

# Modify config_json here before loading.
config_json.update({
    "head_dim": 128,
    "hidden_size": 8,
    "index_head_dim": 128,
    "index_n_heads": 4,
    "moe_intermediate_size": 32,
    "n_routed_experts": 256,
    # "n_shared_experts": 1,
    "num_attention_heads": 8,
    "num_hidden_layers": 7,
    "num_experts_per_tok": 6,
    "num_hash_layers": 3,
    "num_key_value_heads": 1,
    "q_lora_rank": 128,
    "o_lora_rank": 128,
    "o_groups": 8,
    "layer_types": [
        "sliding_attention",
        "sliding_attention",
        "compressed_sparse_attention",
        "heavily_compressed_attention",
        "compressed_sparse_attention",
        "heavily_compressed_attention",
        "compressed_sparse_attention"
    ],
    "compress_ratios": [0, 0, 4, 128, 4, 128, 4, 0],
})
del config_json['quantization_config']
del config_json['expert_dtype']

with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f:
    json.dump(config_json, f, indent=2)

config = AutoConfig.from_pretrained(
    save_folder,
    trust_remote_code=True,
)
assert isinstance(config, DeepseekV4Config)
model = AutoModelForCausalLM.from_config(
    config,
    dtype=torch.bfloat16,
    trust_remote_code=True,
)
if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'):
    model.generation_config = GenerationConfig.from_pretrained(
        source_model_id, trust_remote_code=True,
    )
set_seed(42)
max_width = max(len(name) for name in model.state_dict().keys())
name2size = {name: tensor.numel() * tensor.element_size() for name, tensor in model.state_dict().items()}

print("Top 10 tensor sizes:")
for name, size in heapq.nlargest(10, name2size.items(), key=lambda x: x[1]):
    print(
        name.ljust(max_width),
        f'{size / 1024**2:.2f}MB',
        model.state_dict()[name].shape,
    )

print("Initializing parameters:")
with torch.no_grad():
    for name, p in sorted(model.named_parameters()):
        torch.nn.init.normal_(p, 0, 0.3)
        print(
            name.ljust(max_width),
            f'{name2size[name] / 1024**2:.2f}MB',
            p.shape, p.dtype,
        )
    for name, b in model.named_buffers():
        print(
            name.ljust(max_width),
            f'{b.numel() * b.element_size() / 1024**2:.2f}MB',
            b.shape, b.dtype,
        )
        if name.endswith("e_score_correction_bias"):
            torch.nn.init.normal_(b, 0, 0.3)
        elif name.endswith("tid2eid"):
            b.copy_(torch.rand(
                b.shape[0], config.n_routed_experts, device=b.device,
            ).topk(b.shape[1], dim=-1).indices.to(b.dtype))
        elif name.endswith("inv_freq"):
            pass
        else:
            raise ValueError(f"Unknown buffer: {name}")

model.save_pretrained(save_folder)
with open(f"{save_folder}/config.json", "r", encoding="utf-8") as f:
    saved_config_json = json.load(f)

# vLLM reads old config.json, so we need to modify it to make it compatible.
saved_config_json.pop("mlp_layer_types", None)
saved_config_json["num_hash_layers"] = config_json["num_hash_layers"]
saved_config_json["compress_ratios"] = [0, 0, 4, 128, 4, 128, 4, 0]
rope_parameters = saved_config_json.pop("rope_parameters", None)
saved_config_json["rope_scaling"] = {
    "beta_fast": 32,
    "beta_slow": 1,
    "factor": 16,
    "original_max_position_embeddings": 65536,
    "type": "yarn"
}
saved_config_json["rope_theta"] = 10000
saved_config_json["compress_rope_theta"] = 160000
with open(f"{save_folder}/config.json", "w", encoding="utf-8") as f:
    json.dump(saved_config_json, f, indent=2)

text

DeepseekV4ForCausalLM(
  (model): DeepseekV4Model(
    (embed_tokens): Embedding(129280, 8)
    (layers): ModuleList(
      (0-1): 2 x DeepseekV4DecoderLayer(
        (self_attn): DeepseekV4Attention(
          (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
          (q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
          (q_b_norm): DeepseekV4UnweightedRMSNorm()
          (kv_proj): Linear(in_features=8, out_features=128, bias=False)
          (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
          (o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
        )
        (mlp): DeepseekV4SparseMoeBlock(
          (gate): DeepseekV4HashRouter(
            (score_fn): SqrtSoftplusActivation()
          )
          (experts): DeepseekV4Experts(
            (act_fn): SiLUActivation()
          )
          (shared_experts): DeepseekV4MLP(
            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
            (up_proj): Linear(in_features=8, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=8, bias=False)
            (act_fn): SiLUActivation()
          )
        )
        (input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (attn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
        (ffn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
      )
      (2): DeepseekV4DecoderLayer(
        (self_attn): DeepseekV4Attention(
          (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
          (q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
          (q_b_norm): DeepseekV4UnweightedRMSNorm()
          (kv_proj): Linear(in_features=8, out_features=128, bias=False)
          (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
          (o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
          (compressor): DeepseekV4CSACompressor(
            (kv_proj): Linear(in_features=8, out_features=256, bias=False)
            (gate_proj): Linear(in_features=8, out_features=256, bias=False)
            (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
            (rotary_emb): DeepseekV4RotaryEmbedding()
            (indexer): DeepseekV4Indexer(
              (kv_proj): Linear(in_features=8, out_features=256, bias=False)
              (gate_proj): Linear(in_features=8, out_features=256, bias=False)
              (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
              (q_b_proj): Linear(in_features=128, out_features=512, bias=False)
              (rotary_emb): DeepseekV4RotaryEmbedding()
              (scorer): DeepseekV4IndexerScorer(
                (weights_proj): Linear(in_features=8, out_features=4, bias=False)
              )
            )
          )
        )
        (mlp): DeepseekV4SparseMoeBlock(
          (gate): DeepseekV4HashRouter(
            (score_fn): SqrtSoftplusActivation()
          )
          (experts): DeepseekV4Experts(
            (act_fn): SiLUActivation()
          )
          (shared_experts): DeepseekV4MLP(
            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
            (up_proj): Linear(in_features=8, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=8, bias=False)
            (act_fn): SiLUActivation()
          )
        )
        (input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (attn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
        (ffn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
      )
      (3): DeepseekV4DecoderLayer(
        (self_attn): DeepseekV4Attention(
          (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
          (q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
          (q_b_norm): DeepseekV4UnweightedRMSNorm()
          (kv_proj): Linear(in_features=8, out_features=128, bias=False)
          (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
          (o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
          (compressor): DeepseekV4HCACompressor(
            (kv_proj): Linear(in_features=8, out_features=128, bias=False)
            (gate_proj): Linear(in_features=8, out_features=128, bias=False)
            (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
            (rotary_emb): DeepseekV4RotaryEmbedding()
          )
        )
        (mlp): DeepseekV4SparseMoeBlock(
          (gate): DeepseekV4TopKRouter(
            (score_fn): SqrtSoftplusActivation()
          )
          (experts): DeepseekV4Experts(
            (act_fn): SiLUActivation()
          )
          (shared_experts): DeepseekV4MLP(
            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
            (up_proj): Linear(in_features=8, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=8, bias=False)
            (act_fn): SiLUActivation()
          )
        )
        (input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (attn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
        (ffn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
      )
      (4): DeepseekV4DecoderLayer(
        (self_attn): DeepseekV4Attention(
          (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
          (q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
          (q_b_norm): DeepseekV4UnweightedRMSNorm()
          (kv_proj): Linear(in_features=8, out_features=128, bias=False)
          (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
          (o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
          (compressor): DeepseekV4CSACompressor(
            (kv_proj): Linear(in_features=8, out_features=256, bias=False)
            (gate_proj): Linear(in_features=8, out_features=256, bias=False)
            (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
            (rotary_emb): DeepseekV4RotaryEmbedding()
            (indexer): DeepseekV4Indexer(
              (kv_proj): Linear(in_features=8, out_features=256, bias=False)
              (gate_proj): Linear(in_features=8, out_features=256, bias=False)
              (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
              (q_b_proj): Linear(in_features=128, out_features=512, bias=False)
              (rotary_emb): DeepseekV4RotaryEmbedding()
              (scorer): DeepseekV4IndexerScorer(
                (weights_proj): Linear(in_features=8, out_features=4, bias=False)
              )
            )
          )
        )
        (mlp): DeepseekV4SparseMoeBlock(
          (gate): DeepseekV4TopKRouter(
            (score_fn): SqrtSoftplusActivation()
          )
          (experts): DeepseekV4Experts(
            (act_fn): SiLUActivation()
          )
          (shared_experts): DeepseekV4MLP(
            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
            (up_proj): Linear(in_features=8, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=8, bias=False)
            (act_fn): SiLUActivation()
          )
        )
        (input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (attn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
        (ffn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
      )
      (5): DeepseekV4DecoderLayer(
        (self_attn): DeepseekV4Attention(
          (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
          (q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
          (q_b_norm): DeepseekV4UnweightedRMSNorm()
          (kv_proj): Linear(in_features=8, out_features=128, bias=False)
          (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
          (o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
          (compressor): DeepseekV4HCACompressor(
            (kv_proj): Linear(in_features=8, out_features=128, bias=False)
            (gate_proj): Linear(in_features=8, out_features=128, bias=False)
            (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
            (rotary_emb): DeepseekV4RotaryEmbedding()
          )
        )
        (mlp): DeepseekV4SparseMoeBlock(
          (gate): DeepseekV4TopKRouter(
            (score_fn): SqrtSoftplusActivation()
          )
          (experts): DeepseekV4Experts(
            (act_fn): SiLUActivation()
          )
          (shared_experts): DeepseekV4MLP(
            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
            (up_proj): Linear(in_features=8, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=8, bias=False)
            (act_fn): SiLUActivation()
          )
        )
        (input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (attn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
        (ffn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
      )
      (6): DeepseekV4DecoderLayer(
        (self_attn): DeepseekV4Attention(
          (q_a_proj): Linear(in_features=8, out_features=128, bias=False)
          (q_a_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (q_b_proj): Linear(in_features=128, out_features=1024, bias=False)
          (q_b_norm): DeepseekV4UnweightedRMSNorm()
          (kv_proj): Linear(in_features=8, out_features=128, bias=False)
          (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
          (o_a_proj): DeepseekV4GroupedLinear(in_features=128, out_features=1024, bias=False)
          (o_b_proj): Linear(in_features=1024, out_features=8, bias=False)
          (compressor): DeepseekV4CSACompressor(
            (kv_proj): Linear(in_features=8, out_features=256, bias=False)
            (gate_proj): Linear(in_features=8, out_features=256, bias=False)
            (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
            (rotary_emb): DeepseekV4RotaryEmbedding()
            (indexer): DeepseekV4Indexer(
              (kv_proj): Linear(in_features=8, out_features=256, bias=False)
              (gate_proj): Linear(in_features=8, out_features=256, bias=False)
              (kv_norm): DeepseekV4RMSNorm((128,), eps=1e-06)
              (q_b_proj): Linear(in_features=128, out_features=512, bias=False)
              (rotary_emb): DeepseekV4RotaryEmbedding()
              (scorer): DeepseekV4IndexerScorer(
                (weights_proj): Linear(in_features=8, out_features=4, bias=False)
              )
            )
          )
        )
        (mlp): DeepseekV4SparseMoeBlock(
          (gate): DeepseekV4TopKRouter(
            (score_fn): SqrtSoftplusActivation()
          )
          (experts): DeepseekV4Experts(
            (act_fn): SiLUActivation()
          )
          (shared_experts): DeepseekV4MLP(
            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
            (up_proj): Linear(in_features=8, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=8, bias=False)
            (act_fn): SiLUActivation()
          )
        )
        (input_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (post_attention_layernorm): DeepseekV4RMSNorm((8,), eps=1e-06)
        (attn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
        (ffn_hc): DeepseekV4HyperConnection(
          (input_norm): DeepseekV4UnweightedRMSNorm()
        )
      )
    )
    (norm): DeepseekV4RMSNorm((8,), eps=1e-06)
    (rotary_emb): DeepseekV4RotaryEmbedding()
    (hc_head): DeepseekV4HyperHead(
      (input_norm): DeepseekV4UnweightedRMSNorm()
    )
  )
  (lm_head): Linear(in_features=8, out_features=129280, bias=False)
)

deepseek-v4-bf16-tiny-random

Get help setting up a custom Dedicated Endpoints.

README

Example usage:

Codes to create this repo:

Printing the model:

Test environment:

Change log:

Explore FriendliAI today

README

Example usage:

Codes to create this repo:

Printing the model:

Test environment:

Change log: