merge_method: della_linear
base_model: google/gemma-4-31B-it
dtype: bfloat16
parameters:
epsilon: 0.05
lambda: 1.0
normalize: false
rescale: false
models:
- model: ApocalypseParty/G4-31B-SFT-v5-2-soup
parameters:
density:
- filter: embed_tokens
value: 1.0
- filter: lm_head
value: 1.0
- filter: vision_tower
value: 1.0
- filter: embed_vision
value: 1.0
- value: 0.7
weight:
- filter: embed_tokens
value: 0.0
- filter: embed_vision
value: 0.0
- filter: vision_tower
value: 0.0
- filter: model.language_model.norm.
value: 0.0
- filter: layer_scalar
value: 0.0
- filter: lm_head
value: 0.0
- filter: model.language_model.layers.5.
value: 0.0
- filter: model.language_model.layers.11.
value: 0.0
- filter: model.language_model.layers.17.
value: 0.0
- filter: model.language_model.layers.23.
value: 0.0
- filter: model.language_model.layers.29.
value: 0.0
- filter: model.language_model.layers.35.
value: 0.0
- filter: model.language_model.layers.41.
value: 0.0
- filter: model.language_model.layers.47.
value: 0.0
- filter: model.language_model.layers.53.
value: 0.0
- filter: model.language_model.layers.59.
value: 0.0
- value: [0.20, 0.35, 0.40, 0.35, 0.20]
- model: ApocalypseParty/G4-31B-SFT-Diversity-1-5ep
parameters:
density:
- filter: embed_tokens
value: 1.0
- filter: lm_head
value: 1.0
- filter: vision_tower
value: 1.0
- filter: embed_vision
value: 1.0
- value: 0.6
weight:
- filter: embed_tokens
value: 0.0
- filter: embed_vision
value: 0.0
- filter: vision_tower
value: 0.0
- filter: model.language_model.norm.
value: 0.0
- filter: layer_scalar
value: 0.0
- filter: lm_head
value: 0.0
- filter: model.language_model.layers.5.
value: 0.0
- filter: model.language_model.layers.11.
value: 0.0
- filter: model.language_model.layers.17.
value: 0.0
- filter: model.language_model.layers.23.
value: 0.0
- filter: model.language_model.layers.29.
value: 0.0
- filter: model.language_model.layers.35.
value: 0.0
- filter: model.language_model.layers.41.
value: 0.0
- filter: model.language_model.layers.47.
value: 0.0
- filter: model.language_model.layers.53.
value: 0.0
- filter: model.language_model.layers.59.
value: 0.0
- filter: self_attn
value: 0.0
- value: [0.10, 0.20, 0.25, 0.20, 0.10]
