doth4580 commited on
Commit
dab1188
·
verified ·
1 Parent(s): 63c4e2e

Add configuration_naive_n05_flash.py — Naive-N0.5-Flash EXL3 3.5 bpw

Browse files
Files changed (1) hide show
  1. configuration_naive_n05_flash.py +155 -0
configuration_naive_n05_flash.py ADDED
@@ -0,0 +1,155 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 Naive AI.
2
+ # Copyright 2026 The HuggingFace Inc. team.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ from transformers.configuration_utils import PretrainedConfig
17
+
18
+
19
+ class NaiveN05FlashConfig(PretrainedConfig):
20
+ model_type = "naive_n05_flash"
21
+ keys_to_ignore_at_inference = ["past_key_values"]
22
+ attribute_map = {"num_local_experts": "n_routed_experts"}
23
+
24
+ def __init__(
25
+ self,
26
+ vocab_size: int = 152576,
27
+ hidden_size: int = 4096,
28
+ intermediate_size: int = 16384,
29
+ num_hidden_layers: int = 48,
30
+ num_attention_heads: int = 64,
31
+ num_key_value_heads: int = 4,
32
+ head_dim: int = 192,
33
+ v_head_dim: int = 128,
34
+ swa_num_attention_heads: int = 64,
35
+ swa_num_key_value_heads: int = 8,
36
+ swa_head_dim: int = 192,
37
+ swa_v_head_dim: int = 128,
38
+ hidden_act: str = "silu",
39
+ max_position_embeddings: int = 1048576,
40
+ initializer_range: float = 0.02,
41
+ layernorm_epsilon: float = 1e-5,
42
+ rope_theta: float = 10000000.0,
43
+ swa_rope_theta: float = 10000.0,
44
+ partial_rotary_factor: float = 0.334,
45
+ sliding_window: int = 128,
46
+ hybrid_layer_pattern: list[int] | None = None,
47
+ attention_bias: bool = False,
48
+ attention_projection_layout: str = "split",
49
+ attention_value_scale: float | None = 0.707,
50
+ add_full_attention_sink_bias: bool = False,
51
+ add_swa_attention_sink_bias: bool = True,
52
+ n_routed_experts: int = 256,
53
+ moe_intermediate_size: int = 2048,
54
+ num_experts_per_tok: int = 8,
55
+ routed_scaling_factor: float | None = 1.0,
56
+ scoring_func: str = "sigmoid",
57
+ topk_method: str = "noaux_tc",
58
+ n_group: int = 1,
59
+ topk_group: int = 1,
60
+ norm_topk_prob: bool = True,
61
+ moe_layer_freq: list[int] | None = None,
62
+ enable_dsa: bool = True,
63
+ index_top_k: int = 2048,
64
+ index_head_dim: int = 128,
65
+ index_n_heads: int = 16,
66
+ index_n_kv_heads: int = 1,
67
+ indexer_activation_dtype: str = "fp8_e4m3",
68
+ use_cache: bool = True,
69
+ tie_word_embeddings: bool = False,
70
+ pad_token_id: int | None = 151643,
71
+ eos_token_id: int | list[int] | None = 151645,
72
+ **kwargs,
73
+ ) -> None:
74
+ self.vocab_size = vocab_size
75
+ self.hidden_size = hidden_size
76
+ self.intermediate_size = intermediate_size
77
+ self.num_hidden_layers = num_hidden_layers
78
+ self.num_attention_heads = num_attention_heads
79
+ self.num_key_value_heads = num_key_value_heads
80
+ self.head_dim = head_dim
81
+ self.v_head_dim = v_head_dim
82
+ self.swa_num_attention_heads = swa_num_attention_heads
83
+ self.swa_num_key_value_heads = swa_num_key_value_heads
84
+ self.swa_head_dim = swa_head_dim
85
+ self.swa_v_head_dim = swa_v_head_dim
86
+ self.hidden_act = hidden_act
87
+ self.max_position_embeddings = max_position_embeddings
88
+ self.initializer_range = initializer_range
89
+ self.layernorm_epsilon = layernorm_epsilon
90
+ self.rope_theta = rope_theta
91
+ self.swa_rope_theta = swa_rope_theta
92
+ self.partial_rotary_factor = partial_rotary_factor
93
+ self.sliding_window = sliding_window
94
+ self.hybrid_layer_pattern = hybrid_layer_pattern or [
95
+ int(i != 0 and i % 6 != 5) for i in range(num_hidden_layers)
96
+ ]
97
+ self.attention_bias = attention_bias
98
+ self.attention_projection_layout = attention_projection_layout
99
+ self.attention_value_scale = attention_value_scale
100
+ self.add_full_attention_sink_bias = add_full_attention_sink_bias
101
+ self.add_swa_attention_sink_bias = add_swa_attention_sink_bias
102
+ self.n_routed_experts = n_routed_experts
103
+ self.moe_intermediate_size = moe_intermediate_size
104
+ self.num_experts_per_tok = num_experts_per_tok
105
+ self.routed_scaling_factor = routed_scaling_factor or 1.0
106
+ self.scoring_func = scoring_func
107
+ self.topk_method = topk_method
108
+ self.n_group = n_group
109
+ self.topk_group = topk_group
110
+ self.norm_topk_prob = norm_topk_prob
111
+ self.moe_layer_freq = moe_layer_freq or ([0] + [1] * (num_hidden_layers - 1))
112
+ self.enable_dsa = enable_dsa
113
+ self.index_top_k = index_top_k
114
+ self.index_head_dim = index_head_dim
115
+ self.index_n_heads = index_n_heads
116
+ self.index_n_kv_heads = index_n_kv_heads
117
+ self.indexer_activation_dtype = indexer_activation_dtype
118
+ self.use_cache = use_cache
119
+ super().__init__(
120
+ tie_word_embeddings=tie_word_embeddings, pad_token_id=pad_token_id, eos_token_id=eos_token_id, **kwargs
121
+ )
122
+ self.validate()
123
+
124
+ def validate(self):
125
+ if self.attention_projection_layout != "split" or self.index_n_kv_heads != 1:
126
+ raise ValueError("NaiveN05Flash requires split Q/K/V and one indexer KV head")
127
+ if not self.enable_dsa or self.index_top_k <= 0:
128
+ raise ValueError("NaiveN05Flash requires DSA with a positive index_top_k")
129
+ if self.indexer_activation_dtype not in ("bf16", "fp8_e4m3"):
130
+ raise ValueError("indexer_activation_dtype must be bf16 or fp8_e4m3")
131
+ if self.scoring_func != "sigmoid" or self.topk_method != "noaux_tc":
132
+ raise ValueError("NaiveN05Flash requires sigmoid routing with correction bias")
133
+ if self.n_group != 1 or self.topk_group != 1:
134
+ raise ValueError("NaiveN05Flash uses one expert group")
135
+ for prefix in ("", "swa_"):
136
+ heads = getattr(self, prefix + "num_attention_heads")
137
+ kv_heads = getattr(self, prefix + "num_key_value_heads")
138
+ head_dim = getattr(self, prefix + "head_dim")
139
+ rotary_dim = int(head_dim * self.partial_rotary_factor)
140
+ if kv_heads <= 0 or heads % kv_heads or rotary_dim % 2 or not 0 < rotary_dim <= head_dim:
141
+ raise ValueError("Invalid attention head or rotary dimensions")
142
+ if int(self.head_dim * self.partial_rotary_factor) > self.index_head_dim:
143
+ raise ValueError("Indexer head dimension must cover the rotary dimension")
144
+ if self._attn_implementation not in (None, "eager"):
145
+ raise ValueError("NaiveN05Flash supports eager attention only")
146
+ if len(self.hybrid_layer_pattern) != self.num_hidden_layers:
147
+ raise ValueError("hybrid_layer_pattern must contain one entry per layer")
148
+ if len(self.moe_layer_freq) != self.num_hidden_layers:
149
+ raise ValueError("moe_layer_freq must contain one entry per layer")
150
+ self.layer_types = [
151
+ "sliding_attention" if is_swa else "deepseek_sparse_attention" for is_swa in self.hybrid_layer_pattern
152
+ ]
153
+
154
+
155
+ __all__ = ["NaiveN05FlashConfig"]