🌑 Promethean Dawn 26B-A4B

This is a merge of pre-trained language models created using mergekit-exp.
This model was merged using the moe_della merge method, and combines every ReadyArt 26B model.
architecture: Gemma4ForConditionalGeneration
base_model: B:\26B\google--gemma-4-26B-A4B-it
models:
- model: B:\26B\ReadyArt--Dark-Scarlett-v1.0-26B-A4B
parameters:
weight:
- filter: "lm_head"
value: 1.0
- filter: "attn"
value: 1.0
- filter: "embed_tokens"
value: 1.0
- value: 1.0
density: 0.9
epsilon: 0.09
- model: B:\26B\ReadyArt--For-Her-Darkside-26B-A4B-v1.4
parameters:
weight:
- filter: "lm_head"
value: 1.0
- filter: "attn"
value: 1.0
- filter: "embed_tokens"
value: 1.0
- value: 1.0
density: 0.9
epsilon: 0.09
- model: B:\26B\ReadyArt--Heimdallr-26B-A4B-v0.35-Q8_0
parameters:
weight:
- filter: "lm_head"
value: 1.0
- filter: "attn"
value: 1.0
- filter: "embed_tokens"
value: 1.0
- value: 1.0
density: 0.9
epsilon: 0.09
- model: B:\26B\ReadyArt--Melody1437-26B-A4B-v2.0
parameters:
weight:
- filter: "lm_head"
value: 1.0
- filter: "attn"
value: 1.0
- filter: "embed_tokens"
value: 1.0
- value: 1.0
density: 0.9
epsilon: 0.09
- model: B:\26B\ReadyArt--Omega-Evolution-26B-A4B-v3.0-Q8_0
parameters:
weight:
- filter: "lm_head"
value: 1.0
- filter: "attn"
value: 1.0
- filter: "embed_tokens"
value: 1.0
- value: 1.0
density: 0.9
epsilon: 0.09
- model: B:\26B\ReadyArt--Serenity-26B-A4B
parameters:
weight:
- filter: "lm_head"
value: 1.0
- filter: "attn"
value: 1.0
- filter: "embed_tokens"
value: 1.0
- value: 1.0
density: 0.9
epsilon: 0.09
merge_method: moe_della
parameters:
lambda: 1.0
# normalize: true # deprecated
normalize_weights: true
normalize_router: true
int8_mask: false
rescale: true
router_strategy: della # average # random_init
blend_experts: true
dtype: float32
out_dtype: bfloat16
tokenizer:
source: union
chat_template: auto
name: 🌑 Promethean Dawn 26B-A4B