merge_method: della_linear
base_model: google/gemma-4-31B-it
dtype: bfloat16
parameters:
epsilon: 0.05
lambda: 1.0
normalize: false
rescale: false
models:
- model: ApocalypseParty/G4-31B-SFT-Diversity-1-5ep
parameters:
density:
- filter: embed_tokens
value: 1.0
- filter: lm_head
value: 1.0
- filter: vision_tower
value: 1.0
- filter: embed_vision
value: 1.0
- value: 0.6
weight:
- filter: embed_tokens
value: 0.0
- filter: embed_vision
value: 0.0
- filter: vision_tower
value: 0.0
- filter: model.language_model.norm.
value: 0.0
- filter: layer_scalar
value: 0.0
- filter: lm_head
value: 0.0
- filter: model.language_model.layers.5.
value: 0.0
- filter: model.language_model.layers.11.
value: 0.0
- filter: model.language_model.layers.17.
value: 0.0
- filter: model.language_model.layers.23.
value: 0.0
- filter: model.language_model.layers.29.
value: 0.0
- filter: model.language_model.layers.35.
value: 0.0
- filter: model.language_model.layers.41.
value: 0.0
- filter: model.language_model.layers.47.
value: 0.0
- filter: model.language_model.layers.53.
value: 0.0
- filter: model.language_model.layers.59.
value: 0.0
- value: [0.15, 0.25, 0.30, 0.25, 0.15]
- model: ApocalypseParty/G4-31B-SFT-v5-2-soup
parameters:
density:
- filter: embed_tokens
value: 1.0
- filter: lm_head
value: 1.0
- filter: vision_tower
value: 1.0
- filter: embed_vision
value: 1.0
- value: 0.7
weight:
- filter: embed_tokens
value: 0.0
- filter: embed_vision
value: 0.0
- filter: vision_tower
value: 0.0
- filter: model.language_model.norm.
value: 0.0
- filter: layer_scalar
value: 0.0
- filter: lm_head
value: 0.0
- filter: model.language_model.layers.5.
value: 0.0
- filter: model.language_model.layers.11.
value: 0.0
- filter: model.language_model.layers.17.
value: 0.0
- filter: model.language_model.layers.23.
value: 0.0
- filter: model.language_model.layers.29.
value: 0.0
- filter: model.language_model.layers.35.
value: 0.0
- filter: model.language_model.layers.41.
value: 0.0
- filter: model.language_model.layers.47.
value: 0.0
- filter: model.language_model.layers.53.
value: 0.0
- filter: model.language_model.layers.59.
value: 0.0
- value: [0.12, 0.20, 0.25, 0.20, 0.12]
