lukbl commited on
Commit
b689296
·
verified ·
1 Parent(s): 7474e1f

Upload folder using huggingface_hub

Browse files
chat_template.jinja ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- for message in messages %}
2
+ {{- '<|im_start|>' }
3
+ {%- if message['content'] is string %}
4
+ {{- message['content'] }}
5
+ {%- else %}
6
+ {%- for content in message['content'] %}
7
+ {%- if content['type'] == 'text' %}
8
+ {{- content['text'] }}
9
+ {%- endif %}
10
+ {%- endfor %}
11
+ {%- endif %}
12
+ {{- '<|im_end|>\n' }}
13
+ {%- endfor %}
14
+ {%- if add_generation_prompt %}
15
+ {{- '<|im_start|>' }}
16
+ {%- endif %}
config.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "NaFlexCrossForConditionalGeneration"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "naflex_cross.NaFlexCrossConfig",
7
+ "AutoModelForImageTextToText": "naflex_cross.NaFlexCrossForConditionalGeneration",
8
+ "AutoProcessor": "naflex_processor.NaFlexProcessor"
9
+ },
10
+ "bos_token_id": 2,
11
+ "decoder_config": {
12
+ "_comment_params": "tied_embed=4.1M + 4x(self_attn=197K + cross_attn=229K + swiglu=786K + norms~2K)=4.85M",
13
+ "bos_token_id": 2,
14
+ "eos_token_id": 4,
15
+ "hidden_size": 256,
16
+ "intermediate_size": 1024,
17
+ "max_position_embeddings": 2048,
18
+ "num_attention_heads": 4,
19
+ "num_hidden_layers": 4,
20
+ "num_key_value_heads": 2,
21
+ "pad_token_id": 0,
22
+ "rms_norm_eps": 1e-06,
23
+ "rope_theta": 50000.0,
24
+ "tie_word_embeddings": true,
25
+ "vocab_size": 16000
26
+ },
27
+ "dtype": "bfloat16",
28
+ "eos_token_id": 4,
29
+ "model_type": "naflex_cross",
30
+ "pad_token_id": 5,
31
+ "tie_word_embeddings": true,
32
+ "transformers_version": "5.3.0",
33
+ "use_cache": false,
34
+ "vit_hidden_size": 384,
35
+ "vit_intermediate_size": 1536,
36
+ "vit_norm_eps": 1e-06,
37
+ "vit_num_heads": 6,
38
+ "vit_num_layers": 6,
39
+ "vit_patch_size": [
40
+ 14,
41
+ 7
42
+ ],
43
+ "vit_rope_theta": 10000.0
44
+ }
generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 2,
4
+ "eos_token_id": [
5
+ 4
6
+ ],
7
+ "pad_token_id": 5,
8
+ "transformers_version": "5.3.0"
9
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f57d9f3874799081a88ac91d4a73a91b9482c475216826f305e1fcd74704058
3
+ size 39378464
naflex_cross.py ADDED
@@ -0,0 +1,643 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ NaFlex Cross-Attention VLM — HuggingFace-compatible.
3
+
4
+ Architecture:
5
+ - Encoder: NaFlex ViT — variable-resolution ViT with 2D factored RoPE
6
+ - Decoder: Causal LM (4-6 layers) with per-layer cross-attention to ViT features
7
+
8
+ NaFlex key property: patches carry (row, col) positions so any resolution / aspect
9
+ ratio is encoded without positional interpolation. FA2 dispatches automatically via
10
+ F.scaled_dot_product_attention on CUDA with fp16/bf16 inputs.
11
+
12
+ Default tiny config: ~19.8M parameters total (ViT 10.9M + decoder 8.9M).
13
+ vit: hidden=384, layers=6, heads=6, intermediate=1536, patch=16
14
+ dec: hidden=256, layers=4, heads=4, kv_heads=2, intermediate=1024, vocab=16000
15
+
16
+ Usage:
17
+ from rlatex.models.naflex_cross import NaFlexCrossConfig, NaFlexCrossForConditionalGeneration
18
+ NaFlexCrossConfig.register_for_auto_class()
19
+ NaFlexCrossForConditionalGeneration.register_for_auto_class("AutoModelForImageTextToText")
20
+
21
+ Inputs (forward / generate):
22
+ pixel_values [B, N_max, 3*P*P] padded flattened patches (float)
23
+ patch_positions [B, N_max, 2] (row, col) in patch units (long)
24
+ patch_attention_mask [B, N_max] True for valid patches (bool/float)
25
+ input_ids [B, T]
26
+ labels [B, T] -100 for ignored positions
27
+ """
28
+
29
+ import torch
30
+ import torch.nn as nn
31
+ import torch.nn.functional as F
32
+ from transformers import PretrainedConfig, PreTrainedModel
33
+ from transformers.modeling_outputs import CausalLMOutputWithPast
34
+
35
+
36
+ # ── Config ─────────────────────────────────────────────────────────────────────
37
+
38
+
39
+ class NaFlexCrossConfig(PretrainedConfig):
40
+ model_type = "naflex_cross"
41
+
42
+ def __init__(
43
+ self,
44
+ # NaFlex ViT encoder
45
+ vit_patch_size: int | tuple[int, int] = 16, # (patch_h, patch_w) or single int for square
46
+ vit_hidden_size: int = 384,
47
+ vit_num_layers: int = 6,
48
+ vit_num_heads: int = 6, # head_dim = 384 / 6 = 64
49
+ vit_intermediate_size: int = 1536,
50
+ vit_norm_eps: float = 1e-6,
51
+ vit_rope_theta: float = 10000.0,
52
+ # Decoder
53
+ decoder_config: dict | None = None,
54
+ **kwargs,
55
+ ):
56
+ kwargs.setdefault("tie_word_embeddings", True)
57
+ super().__init__(**kwargs)
58
+ # Normalise to (patch_h, patch_w) tuple; stored as list for JSON serialisability
59
+ if isinstance(vit_patch_size, int):
60
+ vit_patch_size = (vit_patch_size, vit_patch_size)
61
+ self.vit_patch_size = list(vit_patch_size)
62
+ self.vit_hidden_size = vit_hidden_size
63
+ self.vit_num_layers = vit_num_layers
64
+ self.vit_num_heads = vit_num_heads
65
+ self.vit_intermediate_size = vit_intermediate_size
66
+ self.vit_norm_eps = vit_norm_eps
67
+ self.vit_rope_theta = vit_rope_theta
68
+ self.decoder_config = decoder_config or {
69
+ # ~8.9M (incl. shared embed with vocab=16000)
70
+ "hidden_size": 256,
71
+ "num_hidden_layers": 4,
72
+ "num_attention_heads": 4, # head_dim = 256 / 4 = 64
73
+ "num_key_value_heads": 2,
74
+ "intermediate_size": 1024,
75
+ "vocab_size": 16000,
76
+ "max_position_embeddings": 2048,
77
+ "rope_theta": 50000.0,
78
+ "rms_norm_eps": 1e-6,
79
+ "bos_token_id": 2,
80
+ "eos_token_id": 4,
81
+ "pad_token_id": 0,
82
+ "tie_word_embeddings": True,
83
+ }
84
+
85
+ @property
86
+ def vit_patch_h(self) -> int:
87
+ return self.vit_patch_size[0]
88
+
89
+ @property
90
+ def vit_patch_w(self) -> int:
91
+ return self.vit_patch_size[1]
92
+
93
+ @property
94
+ def hidden_size(self) -> int:
95
+ return self.decoder_config.get("hidden_size", 256)
96
+
97
+ @property
98
+ def text_config(self) -> "NaFlexCrossConfig":
99
+ """Compatibility shim — lets code like config.text_config.hidden_size work."""
100
+ return self
101
+
102
+
103
+ # ── Shared primitives ──────────────────────────────────────────────────────────
104
+
105
+
106
+ class RMSNorm(nn.Module):
107
+ def __init__(self, hidden_size: int, eps: float = 1e-6):
108
+ super().__init__()
109
+ self.weight = nn.Parameter(torch.ones(hidden_size))
110
+ self.variance_epsilon = eps
111
+
112
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
113
+ variance = x.pow(2).mean(-1, keepdim=True)
114
+ x = x * torch.rsqrt(variance + self.variance_epsilon)
115
+ return self.weight * x
116
+
117
+
118
+ def _rotate_half_split(x: torch.Tensor) -> torch.Tensor:
119
+ """Rotate each half of x independently: cat(-x2, x1) within each chunk."""
120
+ x1, x2 = x.chunk(2, dim=-1)
121
+ return torch.cat((-x2, x1), dim=-1)
122
+
123
+
124
+ # ── 2D Factored RoPE (for ViT) ───────────────────────────────────────��─────────
125
+
126
+
127
+ class RotaryEmbedding2D(nn.Module):
128
+ """
129
+ Factored 2D RoPE for NaFlex ViT.
130
+
131
+ head_dim is split in two equal halves:
132
+ first head_dim//2 dimensions carry row-axis RoPE
133
+ second head_dim//2 dimensions carry col-axis RoPE
134
+ No interpolation — position indices are raw patch row/col integers.
135
+ """
136
+
137
+ def __init__(self, head_dim: int, theta: float = 10000.0):
138
+ super().__init__()
139
+ assert head_dim % 4 == 0, "head_dim must be divisible by 4 for 2D RoPE"
140
+ freq_dim = head_dim // 4
141
+ inv_freq = 1.0 / (
142
+ theta ** (torch.arange(0, freq_dim * 2, 2, dtype=torch.float32) / (freq_dim * 2))
143
+ )
144
+ self.register_buffer("inv_freq", inv_freq, persistent=False) # [freq_dim]
145
+
146
+ def forward(
147
+ self, positions: torch.Tensor # [B, N, 2] long: (row, col)
148
+ ) -> tuple[torch.Tensor, torch.Tensor]:
149
+ """Returns cos, sin each of shape [B, N, head_dim]."""
150
+ rows = positions[..., 0].float() # [B, N]
151
+ cols = positions[..., 1].float() # [B, N]
152
+
153
+ row_freqs = torch.einsum("bn,d->bnd", rows, self.inv_freq) # [B, N, freq_dim]
154
+ col_freqs = torch.einsum("bn,d->bnd", cols, self.inv_freq) # [B, N, freq_dim]
155
+
156
+ row_emb = torch.cat([row_freqs, row_freqs], dim=-1) # [B, N, head_dim//2]
157
+ col_emb = torch.cat([col_freqs, col_freqs], dim=-1) # [B, N, head_dim//2]
158
+ emb = torch.cat([row_emb, col_emb], dim=-1) # [B, N, head_dim]
159
+ return emb.cos(), emb.sin()
160
+
161
+
162
+ def _apply_rotary_2d(
163
+ q: torch.Tensor, # [B, H, N, head_dim]
164
+ k: torch.Tensor, # [B, H, N, head_dim]
165
+ cos: torch.Tensor, # [B, N, head_dim]
166
+ sin: torch.Tensor, # [B, N, head_dim]
167
+ ) -> tuple[torch.Tensor, torch.Tensor]:
168
+ """Apply 2D factored RoPE: row-half and col-half are rotated separately."""
169
+ half = q.shape[-1] // 2
170
+ cos = cos.unsqueeze(1).to(q.dtype) # [B, 1, N, head_dim]
171
+ sin = sin.unsqueeze(1).to(q.dtype)
172
+
173
+ cos_row, cos_col = cos[..., :half], cos[..., half:]
174
+ sin_row, sin_col = sin[..., :half], sin[..., half:]
175
+
176
+ q_row, q_col = q[..., :half], q[..., half:]
177
+ k_row, k_col = k[..., :half], k[..., half:]
178
+
179
+ q_row = q_row * cos_row + _rotate_half_split(q_row) * sin_row
180
+ q_col = q_col * cos_col + _rotate_half_split(q_col) * sin_col
181
+ k_row = k_row * cos_row + _rotate_half_split(k_row) * sin_row
182
+ k_col = k_col * cos_col + _rotate_half_split(k_col) * sin_col
183
+
184
+ return torch.cat([q_row, q_col], dim=-1), torch.cat([k_row, k_col], dim=-1)
185
+
186
+
187
+ # ── 1D RoPE (for decoder) ─────────────────────────────────────────────────────
188
+
189
+
190
+ class RotaryEmbedding1D(nn.Module):
191
+ def __init__(self, dim: int, max_seq_len: int = 2048, theta: float = 50000.0):
192
+ super().__init__()
193
+ inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
194
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
195
+
196
+ def forward(self, seq_len: int, device: torch.device) -> tuple[torch.Tensor, torch.Tensor]:
197
+ t = torch.arange(seq_len, device=device, dtype=self.inv_freq.dtype)
198
+ freqs = torch.outer(t, self.inv_freq)
199
+ emb = torch.cat([freqs, freqs], dim=-1) # [T, dim]
200
+ return emb.cos(), emb.sin()
201
+
202
+
203
+ def _apply_rotary_1d(
204
+ q: torch.Tensor,
205
+ k: torch.Tensor,
206
+ cos: torch.Tensor, # [1, 1, T, head_dim]
207
+ sin: torch.Tensor,
208
+ ) -> tuple[torch.Tensor, torch.Tensor]:
209
+ cos = cos.to(q.dtype)
210
+ sin = sin.to(q.dtype)
211
+ q = q * cos + _rotate_half_split(q) * sin
212
+ k = k * cos + _rotate_half_split(k) * sin
213
+ return q, k
214
+
215
+
216
+ # ── SwiGLU MLP ────────────────────────────────────────────────────────────────
217
+
218
+
219
+ class SwiGLU(nn.Module):
220
+ def __init__(self, hidden_size: int, intermediate_size: int):
221
+ super().__init__()
222
+ self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
223
+ self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
224
+ self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
225
+
226
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
227
+ return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
228
+
229
+
230
+ # ── NaFlex ViT ────────────────────────────────────────────────────────────────
231
+
232
+
233
+ class NaFlexViTAttention(nn.Module):
234
+ """Full (non-causal) multi-head self-attention with 2D RoPE."""
235
+
236
+ def __init__(self, hidden_size: int, num_heads: int):
237
+ super().__init__()
238
+ self.num_heads = num_heads
239
+ self.head_dim = hidden_size // num_heads
240
+ self.scale = self.head_dim ** -0.5
241
+ self.q_proj = nn.Linear(hidden_size, hidden_size, bias=False)
242
+ self.k_proj = nn.Linear(hidden_size, hidden_size, bias=False)
243
+ self.v_proj = nn.Linear(hidden_size, hidden_size, bias=False)
244
+ self.o_proj = nn.Linear(hidden_size, hidden_size, bias=False)
245
+
246
+ def forward(
247
+ self,
248
+ x: torch.Tensor, # [B, N, D]
249
+ cos: torch.Tensor, # [B, N, head_dim]
250
+ sin: torch.Tensor,
251
+ attention_mask: torch.Tensor | None = None, # [B, 1, 1, N] additive
252
+ ) -> torch.Tensor:
253
+ B, N, _ = x.shape
254
+ q = self.q_proj(x).view(B, N, self.num_heads, self.head_dim).transpose(1, 2)
255
+ k = self.k_proj(x).view(B, N, self.num_heads, self.head_dim).transpose(1, 2)
256
+ v = self.v_proj(x).view(B, N, self.num_heads, self.head_dim).transpose(1, 2)
257
+
258
+ q, k = _apply_rotary_2d(q, k, cos, sin)
259
+
260
+ out = F.scaled_dot_product_attention(
261
+ q, k, v,
262
+ attn_mask=attention_mask,
263
+ is_causal=False,
264
+ scale=self.scale,
265
+ )
266
+ return self.o_proj(out.transpose(1, 2).reshape(B, N, -1))
267
+
268
+
269
+ class NaFlexViTLayer(nn.Module):
270
+ def __init__(self, hidden_size: int, num_heads: int, intermediate_size: int, norm_eps: float):
271
+ super().__init__()
272
+ self.norm1 = RMSNorm(hidden_size, eps=norm_eps)
273
+ self.attn = NaFlexViTAttention(hidden_size, num_heads)
274
+ self.norm2 = RMSNorm(hidden_size, eps=norm_eps)
275
+ self.mlp = nn.Sequential(
276
+ nn.Linear(hidden_size, intermediate_size, bias=False),
277
+ nn.GELU(),
278
+ nn.Linear(intermediate_size, hidden_size, bias=False),
279
+ )
280
+
281
+ def forward(
282
+ self,
283
+ x: torch.Tensor,
284
+ cos: torch.Tensor,
285
+ sin: torch.Tensor,
286
+ attention_mask: torch.Tensor | None = None,
287
+ ) -> torch.Tensor:
288
+ x = x + self.attn(self.norm1(x), cos, sin, attention_mask)
289
+ x = x + self.mlp(self.norm2(x))
290
+ return x
291
+
292
+
293
+ class _ParamGroup:
294
+ """
295
+ Lightweight non-Module grouping of parameters for freeze / LR partitioning.
296
+ train_sft.py resolves sub-modules by name and expects a .parameters() iterable.
297
+ Using a plain object avoids double-registering weights with PyTorch.
298
+ """
299
+ def __init__(self, *modules: nn.Module):
300
+ self._modules = modules
301
+
302
+ def parameters(self):
303
+ seen: set[int] = set()
304
+ for m in self._modules:
305
+ for p in m.parameters():
306
+ if id(p) not in seen:
307
+ seen.add(id(p))
308
+ yield p
309
+
310
+
311
+ class NaFlexViTEncoder(nn.Module):
312
+ """
313
+ NaFlex vision transformer encoder.
314
+
315
+ Accepts variable-resolution images as padded batches of flattened patches.
316
+ Each patch carries its (row, col) grid position for 2D RoPE — no positional
317
+ interpolation needed for arbitrary resolutions.
318
+ """
319
+
320
+ def __init__(self, config: NaFlexCrossConfig):
321
+ super().__init__()
322
+ H = config.vit_hidden_size
323
+ self.patch_embed = nn.Linear(3 * config.vit_patch_h * config.vit_patch_w, H, bias=False)
324
+ self.layers = nn.ModuleList([
325
+ NaFlexViTLayer(
326
+ H,
327
+ config.vit_num_heads,
328
+ config.vit_intermediate_size,
329
+ config.vit_norm_eps,
330
+ )
331
+ for _ in range(config.vit_num_layers)
332
+ ])
333
+ self.norm = RMSNorm(H, eps=config.vit_norm_eps)
334
+ self.rope = RotaryEmbedding2D(
335
+ head_dim=H // config.vit_num_heads,
336
+ theta=config.vit_rope_theta,
337
+ )
338
+
339
+ @property
340
+ def blocks(self) -> nn.ModuleList:
341
+ """Alias for train_sft.py which resolves visual layers as `.blocks`."""
342
+ return self.layers
343
+
344
+ def forward(
345
+ self,
346
+ pixel_values: torch.Tensor, # [B, N, 3*P*P]
347
+ patch_positions: torch.Tensor, # [B, N, 2] long: (row, col)
348
+ patch_attention_mask: torch.Tensor | None = None, # [B, N] bool/float
349
+ ) -> torch.Tensor: # [B, N, H]
350
+ x = self.patch_embed(pixel_values)
351
+ cos, sin = self.rope(patch_positions)
352
+
353
+ # Additive mask: valid positions get 0, padding positions get -inf
354
+ attn_mask = None
355
+ if patch_attention_mask is not None:
356
+ valid = patch_attention_mask.float() # [B, N]
357
+ fill = torch.finfo(x.dtype).min
358
+ attn_mask = (1.0 - valid.unsqueeze(1).unsqueeze(2)) * fill # [B, 1, 1, N]
359
+
360
+ for layer in self.layers:
361
+ x = layer(x, cos, sin, attn_mask)
362
+
363
+ return self.norm(x)
364
+
365
+
366
+ # ── Decoder ───────────────────────────────────────────────────────────────────
367
+
368
+
369
+ class GQAttention(nn.Module):
370
+ """Grouped-query attention for decoder self- and cross-attention."""
371
+
372
+ def __init__(
373
+ self,
374
+ hidden_size: int,
375
+ num_heads: int,
376
+ num_kv_heads: int,
377
+ kv_input_size: int | None = None,
378
+ is_causal: bool = True,
379
+ ):
380
+ super().__init__()
381
+ self.num_heads = num_heads
382
+ self.num_kv_heads = num_kv_heads
383
+ self.num_kv_groups = num_heads // num_kv_heads
384
+ self.head_dim = hidden_size // num_heads
385
+ self.scale = self.head_dim ** -0.5
386
+ self.is_causal = is_causal
387
+
388
+ kv_size = kv_input_size if kv_input_size is not None else hidden_size
389
+ self.q_proj = nn.Linear(hidden_size, num_heads * self.head_dim, bias=False)
390
+ self.k_proj = nn.Linear(kv_size, num_kv_heads * self.head_dim, bias=False)
391
+ self.v_proj = nn.Linear(kv_size, num_kv_heads * self.head_dim, bias=False)
392
+ self.o_proj = nn.Linear(num_heads * self.head_dim, hidden_size, bias=False)
393
+
394
+ def forward(
395
+ self,
396
+ hidden_states: torch.Tensor, # [B, T, D]
397
+ kv_source: torch.Tensor | None = None, # [B, S, D_kv] — None → self-attn
398
+ attention_mask: torch.Tensor | None = None,
399
+ cos: torch.Tensor | None = None, # [T, head_dim] — 1D RoPE for self-attn
400
+ sin: torch.Tensor | None = None,
401
+ ) -> torch.Tensor:
402
+ B, T, _ = hidden_states.shape
403
+ src = hidden_states if kv_source is None else kv_source
404
+ S = src.shape[1]
405
+
406
+ q = self.q_proj(hidden_states).view(B, T, self.num_heads, self.head_dim).transpose(1, 2)
407
+ k = self.k_proj(src).view(B, S, self.num_kv_heads, self.head_dim).transpose(1, 2)
408
+ v = self.v_proj(src).view(B, S, self.num_kv_heads, self.head_dim).transpose(1, 2)
409
+
410
+ if cos is not None and kv_source is None:
411
+ cos_t = cos[:T].unsqueeze(0).unsqueeze(0) # [1, 1, T, head_dim]
412
+ sin_t = sin[:T].unsqueeze(0).unsqueeze(0)
413
+ q, k = _apply_rotary_1d(q, k, cos_t, sin_t)
414
+
415
+ if self.num_kv_groups > 1:
416
+ k = k.unsqueeze(2).expand(-1, -1, self.num_kv_groups, -1, -1).reshape(
417
+ B, self.num_heads, S, self.head_dim
418
+ )
419
+ v = v.unsqueeze(2).expand(-1, -1, self.num_kv_groups, -1, -1).reshape(
420
+ B, self.num_heads, S, self.head_dim
421
+ )
422
+
423
+ out = F.scaled_dot_product_attention(
424
+ q, k, v,
425
+ attn_mask=attention_mask,
426
+ is_causal=(self.is_causal and kv_source is None and attention_mask is None),
427
+ scale=self.scale,
428
+ )
429
+ return self.o_proj(out.transpose(1, 2).reshape(B, T, -1))
430
+
431
+
432
+ class NaFlexDecoderLayer(nn.Module):
433
+ def __init__(self, dcfg: dict, vit_hidden: int):
434
+ super().__init__()
435
+ H = dcfg["hidden_size"]
436
+ num_heads = dcfg["num_attention_heads"]
437
+ num_kv_heads = dcfg["num_key_value_heads"]
438
+ eps = dcfg["rms_norm_eps"]
439
+
440
+ self.norm1 = RMSNorm(H, eps=eps)
441
+ self.self_attn = GQAttention(H, num_heads, num_kv_heads, is_causal=True)
442
+
443
+ self.norm_cross = RMSNorm(H, eps=eps)
444
+ self.cross_attn = GQAttention(
445
+ H, num_heads, num_kv_heads, kv_input_size=vit_hidden, is_causal=False
446
+ )
447
+
448
+ self.norm2 = RMSNorm(H, eps=eps)
449
+ self.mlp = SwiGLU(H, dcfg["intermediate_size"])
450
+
451
+ def forward(
452
+ self,
453
+ x: torch.Tensor,
454
+ vision_feats: torch.Tensor,
455
+ vision_mask: torch.Tensor | None = None,
456
+ cos: torch.Tensor | None = None,
457
+ sin: torch.Tensor | None = None,
458
+ ) -> torch.Tensor:
459
+ x = x + self.self_attn(self.norm1(x), cos=cos, sin=sin)
460
+ x = x + self.cross_attn(self.norm_cross(x), kv_source=vision_feats, attention_mask=vision_mask)
461
+ x = x + self.mlp(self.norm2(x))
462
+ return x
463
+
464
+
465
+ # ── Full model ─────────────────────────────────────────────────────────────────
466
+
467
+
468
+ class NaFlexCrossForConditionalGeneration(PreTrainedModel):
469
+ """
470
+ NaFlex VLM: variable-resolution ViT encoder + causal cross-attention decoder.
471
+
472
+ Compatible with HuggingFace Trainer / SFTTrainer.
473
+ FA2 is used automatically by F.scaled_dot_product_attention when inputs are
474
+ fp16/bf16 on CUDA (no extra flags needed).
475
+ Gradient checkpointing is enabled with model.gradient_checkpointing_enable().
476
+ """
477
+
478
+ config_class = NaFlexCrossConfig
479
+ main_input_name = "input_ids"
480
+ supports_gradient_checkpointing = True
481
+ _supports_flash_attn_2 = True
482
+ _tied_weights_keys = {"lm_head.weight": "embed_tokens.weight"}
483
+
484
+ def __init__(self, config: NaFlexCrossConfig):
485
+ super().__init__(config)
486
+ dcfg = config.decoder_config
487
+
488
+ # Vision encoder
489
+ self.visual = NaFlexViTEncoder(config)
490
+
491
+ # Decoder
492
+ H = dcfg["hidden_size"]
493
+ vocab = dcfg["vocab_size"]
494
+ pad_id = dcfg.get("pad_token_id", 0)
495
+
496
+ self.embed_tokens = nn.Embedding(vocab, H, padding_idx=pad_id)
497
+ self.layers = nn.ModuleList([
498
+ NaFlexDecoderLayer(dcfg, vit_hidden=config.vit_hidden_size)
499
+ for _ in range(dcfg["num_hidden_layers"])
500
+ ])
501
+ self.norm = RMSNorm(H, eps=dcfg["rms_norm_eps"])
502
+ self.lm_head = nn.Linear(H, vocab, bias=False)
503
+
504
+ if dcfg.get("tie_word_embeddings", True):
505
+ self.lm_head.weight = self.embed_tokens.weight
506
+
507
+ self.rotary_emb = RotaryEmbedding1D(
508
+ dim=H // dcfg["num_attention_heads"],
509
+ max_seq_len=dcfg["max_position_embeddings"],
510
+ theta=dcfg.get("rope_theta", 50000.0),
511
+ )
512
+
513
+ self.gradient_checkpointing = False
514
+ self.post_init()
515
+
516
+ def get_input_embeddings(self) -> nn.Embedding:
517
+ return self.embed_tokens
518
+
519
+ def set_input_embeddings(self, value: nn.Embedding):
520
+ self.embed_tokens = value
521
+
522
+ def get_output_embeddings(self) -> nn.Linear:
523
+ return self.lm_head
524
+
525
+ def set_output_embeddings(self, value: nn.Linear):
526
+ self.lm_head = value
527
+
528
+ @property
529
+ def language_model(self) -> _ParamGroup:
530
+ """
531
+ Pseudo-module grouping the decoder components.
532
+ Lets train_sft.py resolve the LM side for freeze / per-component LR /
533
+ gradient norm logging without double-registering weights.
534
+ """
535
+ return _ParamGroup(self.embed_tokens, self.layers, self.norm, self.lm_head)
536
+
537
+ def tie_weights(self, **kwargs):
538
+ if self.config.tie_word_embeddings:
539
+ self.lm_head.weight = self.embed_tokens.weight
540
+
541
+ def _encode_images(
542
+ self,
543
+ pixel_values: torch.Tensor,
544
+ patch_positions: torch.Tensor,
545
+ patch_attention_mask: torch.Tensor | None,
546
+ ) -> tuple[torch.Tensor, torch.Tensor]:
547
+ """Encode images and return (vision_feats [B,N,D], vision_mask [B,1,1,N])."""
548
+ vision_feats = self.visual(pixel_values, patch_positions, patch_attention_mask)
549
+
550
+ vision_mask = None
551
+ if patch_attention_mask is not None:
552
+ fill = torch.finfo(vision_feats.dtype).min
553
+ valid = patch_attention_mask.float()
554
+ vision_mask = (1.0 - valid.unsqueeze(1).unsqueeze(2)) * fill # [B, 1, 1, N]
555
+
556
+ return vision_feats, vision_mask
557
+
558
+ def forward(
559
+ self,
560
+ input_ids: torch.Tensor,
561
+ pixel_values: torch.Tensor,
562
+ patch_positions: torch.Tensor,
563
+ patch_attention_mask: torch.Tensor | None = None,
564
+ attention_mask: torch.Tensor | None = None,
565
+ labels: torch.Tensor | None = None,
566
+ # Precomputed vision features (fast generation: encode once, decode many)
567
+ vision_feats: torch.Tensor | None = None,
568
+ vision_mask: torch.Tensor | None = None,
569
+ **kwargs,
570
+ ) -> CausalLMOutputWithPast:
571
+ if vision_feats is None:
572
+ # Cast pixel_values to model dtype (processor always returns float32)
573
+ pixel_values = pixel_values.to(self.visual.patch_embed.weight.dtype)
574
+ vision_feats, vision_mask = self._encode_images(
575
+ pixel_values, patch_positions, patch_attention_mask
576
+ )
577
+
578
+ x = self.embed_tokens(input_ids)
579
+ cos, sin = self.rotary_emb(input_ids.shape[1], device=x.device)
580
+
581
+ for layer in self.layers:
582
+ if self.gradient_checkpointing and self.training:
583
+ x = torch.utils.checkpoint.checkpoint(
584
+ layer, x, vision_feats, vision_mask, cos, sin, use_reentrant=False
585
+ )
586
+ else:
587
+ x = layer(x, vision_feats=vision_feats, vision_mask=vision_mask, cos=cos, sin=sin)
588
+
589
+ x = self.norm(x)
590
+ logits = self.lm_head(x)
591
+
592
+ loss = None
593
+ if labels is not None:
594
+ loss = F.cross_entropy(
595
+ logits[:, :-1].reshape(-1, logits.shape[-1]),
596
+ labels[:, 1:].reshape(-1),
597
+ ignore_index=-100,
598
+ )
599
+
600
+ return CausalLMOutputWithPast(loss=loss, logits=logits)
601
+
602
+ @torch.inference_mode()
603
+ def generate(
604
+ self,
605
+ pixel_values: torch.Tensor,
606
+ patch_positions: torch.Tensor,
607
+ patch_attention_mask: torch.Tensor | None = None,
608
+ input_ids: torch.Tensor | None = None, # optional text prompt
609
+ max_new_tokens: int = 256,
610
+ bos_token_id: int | None = None,
611
+ eos_token_id: int | None = None,
612
+ **kwargs,
613
+ ) -> torch.Tensor:
614
+ """
615
+ Greedy decode. Returns the full sequence (prompt + generated tokens) so
616
+ that callers can slice off the prompt with ``output[:, input_ids.shape[1]:]``.
617
+ If ``input_ids`` is not provided, generation starts from BOS.
618
+ """
619
+ dcfg = self.config.decoder_config
620
+ bos = bos_token_id or dcfg.get("bos_token_id", 2)
621
+ eos = eos_token_id or dcfg.get("eos_token_id", 4)
622
+
623
+ pixel_values = pixel_values.to(self.visual.patch_embed.weight.dtype)
624
+ vision_feats, vision_mask = self._encode_images(
625
+ pixel_values, patch_positions, patch_attention_mask
626
+ )
627
+
628
+ if input_ids is not None:
629
+ ids = input_ids
630
+ else:
631
+ ids = pixel_values.new_full((pixel_values.shape[0], 1), bos, dtype=torch.long)
632
+
633
+ for _ in range(max_new_tokens):
634
+ out = self.forward(
635
+ ids, pixel_values, patch_positions, patch_attention_mask,
636
+ vision_feats=vision_feats, vision_mask=vision_mask,
637
+ )
638
+ next_tok = out.logits[:, -1].argmax(dim=-1, keepdim=True)
639
+ ids = torch.cat([ids, next_tok], dim=1)
640
+ if (next_tok == eos).all():
641
+ break
642
+
643
+ return ids # full sequence (prompt + generated)
naflex_processor.py ADDED
@@ -0,0 +1,342 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ NaFlexProcessor — image + text processor for NaFlexCrossForConditionalGeneration.
3
+
4
+ Image pipeline:
5
+ 1. Convert to RGB
6
+ 2. Resize so the patch count stays within `max_num_patches`, snapping H and W
7
+ to multiples of patch_h / patch_w respectively (preserves aspect ratio).
8
+ 3. Normalise with per-channel mean/std.
9
+ 4. Extract patches row-major; record (row, col) integer position of each patch.
10
+ 5. Pad across the batch to the longest sequence; return patch_attention_mask.
11
+
12
+ Text pipeline:
13
+ Standard HuggingFace tokenizer with padding / truncation.
14
+
15
+ Usage:
16
+ processor = NaFlexProcessor.from_pretrained("checkpoints/qwen2vl-24m",
17
+ patch_size=(16, 16))
18
+ batch = processor(
19
+ text=texts,
20
+ images=image_inputs,
21
+ return_tensors="pt",
22
+ padding="longest",
23
+ truncation=True,
24
+ max_length=512,
25
+ )
26
+ # batch keys: input_ids, attention_mask,
27
+ # pixel_values, patch_positions, patch_attention_mask
28
+ """
29
+
30
+ import math
31
+ from typing import List, Optional, Union
32
+
33
+ import numpy as np
34
+ import torch
35
+ from PIL import Image
36
+ from transformers import AutoTokenizer, BatchFeature, ProcessorMixin
37
+ from transformers.tokenization_utils_base import PaddingStrategy, TruncationStrategy
38
+
39
+
40
+ _DEFAULT_MEAN = (0.7931, 0.7931, 0.7931)
41
+ _DEFAULT_STD = (0.1738, 0.1738, 0.1738)
42
+
43
+ # Cross-attention template: images are encoded by the ViT, NOT injected as tokens.
44
+ # Image content blocks are silently dropped; only text reaches the tokenizer.
45
+ _DEFAULT_CHAT_TEMPLATE = (
46
+ "{%- for message in messages %}\n"
47
+ "{{- '<|im_start|>' }\n"
48
+ "{%- if message['content'] is string %}\n"
49
+ "{{- message['content'] }}\n"
50
+ "{%- else %}\n"
51
+ "{%- for content in message['content'] %}\n"
52
+ "{%- if content['type'] == 'text' %}\n"
53
+ "{{- content['text'] }}\n"
54
+ "{%- endif %}\n"
55
+ "{%- endfor %}\n"
56
+ "{%- endif %}\n"
57
+ "{{- '<|im_end|>\\n' }}\n"
58
+ "{%- endfor %}\n"
59
+ "{%- if add_generation_prompt %}\n"
60
+ "{{- '<|im_start|>' }}\n"
61
+ "{%- endif %}\n"
62
+ )
63
+
64
+
65
+ class NaFlexProcessor(ProcessorMixin):
66
+ """
67
+ Processor for NaFlexCrossForConditionalGeneration.
68
+
69
+ Args:
70
+ tokenizer: Any HuggingFace tokenizer.
71
+ patch_size: (patch_h, patch_w) or single int for square patches.
72
+ max_num_patches: Maximum patches per image (controls resolution budget).
73
+ image_mean: Per-channel normalisation mean (C,).
74
+ image_std: Per-channel normalisation std (C,).
75
+ """
76
+
77
+ attributes = ["tokenizer"]
78
+ tokenizer_class = "AutoTokenizer"
79
+ # Sentinel: NaFlex has no image tokens in the text sequence.
80
+ # train_sft.py uses this to mask image tokens in labels; -1 is never a real token ID.
81
+ image_token_id: int = -1
82
+
83
+ def __init__(
84
+ self,
85
+ tokenizer,
86
+ patch_size: Union[int, tuple] = 16,
87
+ max_num_patches: int = 1024,
88
+ image_mean: tuple = _DEFAULT_MEAN,
89
+ image_std: tuple = _DEFAULT_STD,
90
+ chat_template: Optional[str] = None,
91
+ ):
92
+ if isinstance(patch_size, int):
93
+ patch_size = (patch_size, patch_size)
94
+ self.patch_size = list(patch_size) # serialised as [patch_h, patch_w]
95
+ self.patch_h, self.patch_w = patch_size
96
+ self.max_num_patches = max_num_patches
97
+ self.image_mean = np.array(image_mean, dtype=np.float32)
98
+ self.image_std = np.array(image_std, dtype=np.float32)
99
+ # ProcessorMixin stores chat_template as its own attribute via kwargs
100
+ super().__init__(tokenizer, chat_template=chat_template or _DEFAULT_CHAT_TEMPLATE)
101
+
102
+ # ── Save / load ───────────────────────────────────────────────────────────
103
+
104
+ def save_pretrained(self, save_directory: str, **kwargs):
105
+ """Save processor config, tokenizer, chat template, and a copy of this module."""
106
+ import json, shutil, os
107
+
108
+ # Always enforce the NaFlex cross-attention template (not the early-fusion one
109
+ # that may have been inherited from a Qwen2VL tokenizer source)
110
+ if self.tokenizer.chat_template != _DEFAULT_CHAT_TEMPLATE:
111
+ self.tokenizer.chat_template = _DEFAULT_CHAT_TEMPLATE
112
+
113
+ super().save_pretrained(save_directory, **kwargs)
114
+
115
+ # Write chat_template.jinja and also stamp it directly into tokenizer_config.json.
116
+ # tokenizer.save_pretrained() only writes chat_template there if the tokenizer
117
+ # was originally constructed with it, so we patch it ourselves.
118
+ with open(os.path.join(save_directory, "chat_template.jinja"), "w") as f:
119
+ f.write(self.tokenizer.chat_template)
120
+ tok_cfg_path = os.path.join(save_directory, "tokenizer_config.json")
121
+ if os.path.isfile(tok_cfg_path):
122
+ with open(tok_cfg_path) as f:
123
+ tok_cfg = json.load(f)
124
+ tok_cfg["chat_template"] = self.tokenizer.chat_template
125
+ with open(tok_cfg_path, "w") as f:
126
+ json.dump(tok_cfg, f, indent=2)
127
+
128
+ # Overwrite processor_config.json with all image config fields
129
+ cfg_path = os.path.join(save_directory, "processor_config.json")
130
+ with open(cfg_path) as f:
131
+ cfg = json.load(f)
132
+ cfg["patch_size"] = self.patch_size
133
+ cfg["max_num_patches"] = self.max_num_patches
134
+ cfg["image_mean"] = self.image_mean.tolist()
135
+ cfg["image_std"] = self.image_std.tolist()
136
+ cfg["processor_class"] = "NaFlexProcessor"
137
+ with open(cfg_path, "w") as f:
138
+ json.dump(cfg, f, indent=2)
139
+
140
+ # Copy module file so AutoProcessor can load with trust_remote_code=True
141
+ src = os.path.join(os.path.dirname(__file__), "naflex_processor.py")
142
+ shutil.copy(src, os.path.join(save_directory, "naflex_processor.py"))
143
+
144
+ # Add AutoProcessor entry to config.json auto_map
145
+ main_cfg_path = os.path.join(save_directory, "config.json")
146
+ if os.path.exists(main_cfg_path):
147
+ with open(main_cfg_path) as f:
148
+ main_cfg = json.load(f)
149
+ main_cfg.setdefault("auto_map", {})
150
+ main_cfg["auto_map"]["AutoProcessor"] = "naflex_processor.NaFlexProcessor"
151
+ with open(main_cfg_path, "w") as f:
152
+ json.dump(main_cfg, f, indent=2)
153
+
154
+ @classmethod
155
+ def from_pretrained(
156
+ cls,
157
+ pretrained_model_name_or_path: str,
158
+ patch_size: Union[int, tuple, None] = None,
159
+ max_num_patches: Optional[int] = None,
160
+ image_mean: Optional[tuple] = None,
161
+ image_std: Optional[tuple] = None,
162
+ **kwargs,
163
+ ) -> "NaFlexProcessor":
164
+ import json, os
165
+ # Strip kwargs that belong to AutoProcessor/tokenizer infrastructure
166
+ kwargs.pop("use_fast", None)
167
+ trust_remote_code = kwargs.pop("trust_remote_code", None)
168
+
169
+ # Read saved image config (if loading from a checkpoint directory)
170
+ saved = {}
171
+ cfg_path = os.path.join(pretrained_model_name_or_path, "processor_config.json")
172
+ if os.path.isfile(cfg_path):
173
+ with open(cfg_path) as f:
174
+ saved = json.load(f)
175
+
176
+ tokenizer = AutoTokenizer.from_pretrained(
177
+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
178
+ )
179
+
180
+ # Load chat template: checkpoint file takes priority, then built-in default.
181
+ # This overwrites any early-fusion template inherited from a Qwen2VL tokenizer.
182
+ template_path = os.path.join(pretrained_model_name_or_path, "chat_template.jinja")
183
+ if os.path.isfile(template_path):
184
+ with open(template_path) as f:
185
+ chat_template = f.read()
186
+ else:
187
+ chat_template = _DEFAULT_CHAT_TEMPLATE
188
+
189
+ # Fall back to vit_patch_size from model config.json if not in processor_config
190
+ if patch_size is None and "patch_size" not in saved:
191
+ model_cfg_path = os.path.join(pretrained_model_name_or_path, "config.json")
192
+ if os.path.isfile(model_cfg_path):
193
+ with open(model_cfg_path) as f:
194
+ model_cfg = json.load(f)
195
+ if "vit_patch_size" in model_cfg:
196
+ saved["patch_size"] = model_cfg["vit_patch_size"]
197
+
198
+ return cls(
199
+ tokenizer=tokenizer,
200
+ chat_template=chat_template,
201
+ patch_size=patch_size or saved.get("patch_size", [16, 16]),
202
+ max_num_patches=max_num_patches or saved.get("max_num_patches", 1024),
203
+ image_mean=image_mean or saved.get("image_mean", _DEFAULT_MEAN),
204
+ image_std=image_std or saved.get("image_std", _DEFAULT_STD),
205
+ )
206
+
207
+ # ── Compatibility shim ────────────────────────────────────────────────────
208
+
209
+ @property
210
+ def image_processor(self):
211
+ """
212
+ Shim for code that expects a Qwen2VL-style image_processor (e.g. TokenBudgetDataset).
213
+ NaFlex encodes images via cross-attention ViT — they do NOT add tokens to the text
214
+ sequence, so max_pixels / min_pixels are set to signal zero image-token overhead.
215
+ """
216
+ patch_pixels = self.patch_h * self.patch_w
217
+ max_pixels = self.max_num_patches * patch_pixels
218
+
219
+ class _Shim:
220
+ pass
221
+
222
+ shim = _Shim()
223
+ shim.max_pixels = max_pixels
224
+ shim.min_pixels = patch_pixels
225
+ # Signal to any caller that images don't contribute to the text token count
226
+ shim.image_tokens_in_text = False
227
+ return shim
228
+
229
+ # ── Image helpers ─────────────────────────────────────────────────────────
230
+
231
+ def _resize(self, img: Image.Image, max_num_patches: Optional[int] = None) -> Image.Image:
232
+ """
233
+ Resize image so that:
234
+ - H is a multiple of patch_h, W is a multiple of patch_w
235
+ - patch count <= max_num_patches (or self.max_num_patches if not given)
236
+ - aspect ratio is preserved as closely as possible
237
+ """
238
+ max_n = max_num_patches if max_num_patches is not None else self.max_num_patches
239
+ W_orig, H_orig = img.size
240
+ # Target: scale uniformly so total patches == max_n
241
+ # patches = ceil(H/P_h) * ceil(W/P_w) ≈ (H*W) / (P_h*P_w)
242
+ area_per_patch = self.patch_h * self.patch_w
243
+ scale = math.sqrt(max_n * area_per_patch / (H_orig * W_orig))
244
+ # Round to nearest patch grid boundary
245
+ H_new = max(self.patch_h, round(H_orig * scale / self.patch_h) * self.patch_h)
246
+ W_new = max(self.patch_w, round(W_orig * scale / self.patch_w) * self.patch_w)
247
+ # If already within budget, only snap to grid without upscaling
248
+ if H_orig <= H_new and W_orig <= W_new:
249
+ H_new = max(self.patch_h, round(H_orig / self.patch_h) * self.patch_h)
250
+ W_new = max(self.patch_w, round(W_orig / self.patch_w) * self.patch_w)
251
+ return img.resize((W_new, H_new), Image.BICUBIC)
252
+
253
+ def _patchify(
254
+ self, img: Image.Image, max_num_patches: Optional[int] = None
255
+ ) -> tuple[np.ndarray, np.ndarray]:
256
+ """
257
+ Returns:
258
+ patches: [N, 3 * patch_h * patch_w] float32
259
+ positions: [N, 2] int32 (row, col)
260
+ """
261
+ img = img.convert("RGB")
262
+ img = self._resize(img, max_num_patches=max_num_patches)
263
+ arr = np.array(img, dtype=np.float32) / 255.0 # [H, W, 3]
264
+ arr = (arr - self.image_mean) / self.image_std # normalise
265
+
266
+ H, W, _ = arr.shape
267
+ n_rows = H // self.patch_h
268
+ n_cols = W // self.patch_w
269
+ # Reshape to [n_rows, n_cols, patch_h, patch_w, 3] then flatten per patch
270
+ arr = arr.reshape(n_rows, self.patch_h, n_cols, self.patch_w, 3)
271
+ arr = arr.transpose(0, 2, 1, 3, 4) # [n_rows, n_cols, P_h, P_w, 3]
272
+ patches = arr.reshape(n_rows * n_cols, self.patch_h * self.patch_w * 3) # channels-last order
273
+
274
+ rows, cols = np.meshgrid(np.arange(n_rows), np.arange(n_cols), indexing="ij")
275
+ positions = np.stack([rows.ravel(), cols.ravel()], axis=-1).astype(np.int32)
276
+
277
+ return patches, positions
278
+
279
+ # ── Main __call__ ─────────────────────────────────────────────────────────
280
+
281
+ def __call__(
282
+ self,
283
+ text: Union[str, List[str], None] = None,
284
+ images: Union[Image.Image, List[Image.Image], None] = None,
285
+ return_tensors: Optional[str] = "pt",
286
+ padding: Union[bool, str, PaddingStrategy] = False,
287
+ truncation: Union[bool, str, TruncationStrategy] = False,
288
+ max_length: Optional[int] = None,
289
+ max_num_patches: Optional[int] = None,
290
+ **kwargs,
291
+ ) -> BatchFeature:
292
+ if text is None and images is None:
293
+ raise ValueError("At least one of `text` or `images` must be provided.")
294
+
295
+ # ── Text ──────────────────────────────────────────────────────────────
296
+ encoding = {}
297
+ if text is not None:
298
+ text_enc = self.tokenizer(
299
+ text,
300
+ padding=padding,
301
+ truncation=truncation,
302
+ max_length=max_length,
303
+ return_tensors=return_tensors,
304
+ **kwargs,
305
+ )
306
+ encoding.update(text_enc)
307
+
308
+ # ── Images ────────────────────────────────────────────────────────────
309
+ if images is not None:
310
+ if isinstance(images, Image.Image):
311
+ images = [images]
312
+
313
+ all_patches, all_positions = [], []
314
+ for img in images:
315
+ patches, positions = self._patchify(img, max_num_patches=max_num_patches)
316
+ all_patches.append(patches)
317
+ all_positions.append(positions)
318
+
319
+ # Pad to max N in batch
320
+ max_n = max(p.shape[0] for p in all_patches)
321
+ patch_dim = all_patches[0].shape[1]
322
+
323
+ padded_patches = np.zeros((len(images), max_n, patch_dim), dtype=np.float32)
324
+ padded_positions = np.zeros((len(images), max_n, 2), dtype=np.int32)
325
+ patch_attn_mask = np.zeros((len(images), max_n), dtype=np.bool_)
326
+
327
+ for i, (p, pos) in enumerate(zip(all_patches, all_positions)):
328
+ n = p.shape[0]
329
+ padded_patches[i, :n] = p
330
+ padded_positions[i, :n] = pos
331
+ patch_attn_mask[i, :n] = True
332
+
333
+ if return_tensors == "pt":
334
+ encoding["pixel_values"] = torch.from_numpy(padded_patches)
335
+ encoding["patch_positions"] = torch.from_numpy(padded_positions).long()
336
+ encoding["patch_attention_mask"] = torch.from_numpy(patch_attn_mask)
337
+ else:
338
+ encoding["pixel_values"] = padded_patches
339
+ encoding["patch_positions"] = padded_positions
340
+ encoding["patch_attention_mask"] = patch_attn_mask
341
+
342
+ return BatchFeature(data=encoding, tensor_type=None)
optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59fc3843edd0919848722667405f25d3cfbe4892864d52f88fd26c842773d59f
3
+ size 39432523
processor_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_map": {
3
+ "AutoProcessor": "naflex_processor.NaFlexProcessor"
4
+ },
5
+ "image_mean": [
6
+ 0.7930999994277954,
7
+ 0.7930999994277954,
8
+ 0.7930999994277954
9
+ ],
10
+ "image_std": [
11
+ 0.1738000065088272,
12
+ 0.1738000065088272,
13
+ 0.1738000065088272
14
+ ],
15
+ "max_num_patches": 1024,
16
+ "patch_size": [
17
+ 14,
18
+ 7
19
+ ],
20
+ "processor_class": "NaFlexProcessor"
21
+ }
rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1a0e5159c27f95ddf941e7fdd8971de9b1bf884948c2057943cee9afa8da436a
3
+ size 14645
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9df532fb062f922e323a8e2a1e905258b664b5e86dd48f4d76284c7b79df6901
3
+ size 1785
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": null,
3
+ "auto_map": {
4
+ "AutoProcessor": "naflex_processor.NaFlexProcessor"
5
+ },
6
+ "backend": "tokenizers",
7
+ "bos_token": "<|im_start|>",
8
+ "clean_up_tokenization_spaces": false,
9
+ "eos_token": "<|im_end|>",
10
+ "errors": "replace",
11
+ "is_local": true,
12
+ "max_length": 512,
13
+ "model_max_length": 131072,
14
+ "pad_to_multiple_of": null,
15
+ "pad_token": "<|endoftext|>",
16
+ "pad_token_type_id": 0,
17
+ "padding_side": "left",
18
+ "processor_class": "NaFlexProcessor",
19
+ "split_special_tokens": false,
20
+ "stride": 0,
21
+ "tokenizer_class": "Qwen2Tokenizer",
22
+ "truncation_side": "right",
23
+ "truncation_strategy": "longest_first",
24
+ "unk_token": null,
25
+ "chat_template": "{%- for message in messages %}\n{{- '<|im_start|>' }\n{%- if message['content'] is string %}\n{{- message['content'] }}\n{%- else %}\n{%- for content in message['content'] %}\n{%- if content['type'] == 'text' %}\n{{- content['text'] }}\n{%- endif %}\n{%- endfor %}\n{%- endif %}\n{{- '<|im_end|>\\n' }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n{{- '<|im_start|>' }}\n{%- endif %}\n"
26
+ }
trainer_state.json ADDED
The diff for this file is too large to render. See raw diff
 
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:898695e4b32a5cd2cb452883a79955464695ac9169096c380fcf0ce1a88b737b
3
+ size 5777