Upload folder using huggingface_hub
Browse files- chat_template.jinja +16 -0
- config.json +44 -0
- generation_config.json +9 -0
- model.safetensors +3 -0
- naflex_cross.py +643 -0
- naflex_processor.py +342 -0
- optimizer.pt +3 -0
- processor_config.json +21 -0
- rng_state.pth +3 -0
- scheduler.pt +3 -0
- tokenizer.json +0 -0
- tokenizer_config.json +26 -0
- trainer_state.json +0 -0
- training_args.bin +3 -0
chat_template.jinja
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- for message in messages %}
|
| 2 |
+
{{- '<|im_start|>' }
|
| 3 |
+
{%- if message['content'] is string %}
|
| 4 |
+
{{- message['content'] }}
|
| 5 |
+
{%- else %}
|
| 6 |
+
{%- for content in message['content'] %}
|
| 7 |
+
{%- if content['type'] == 'text' %}
|
| 8 |
+
{{- content['text'] }}
|
| 9 |
+
{%- endif %}
|
| 10 |
+
{%- endfor %}
|
| 11 |
+
{%- endif %}
|
| 12 |
+
{{- '<|im_end|>\n' }}
|
| 13 |
+
{%- endfor %}
|
| 14 |
+
{%- if add_generation_prompt %}
|
| 15 |
+
{{- '<|im_start|>' }}
|
| 16 |
+
{%- endif %}
|
config.json
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"NaFlexCrossForConditionalGeneration"
|
| 4 |
+
],
|
| 5 |
+
"auto_map": {
|
| 6 |
+
"AutoConfig": "naflex_cross.NaFlexCrossConfig",
|
| 7 |
+
"AutoModelForImageTextToText": "naflex_cross.NaFlexCrossForConditionalGeneration",
|
| 8 |
+
"AutoProcessor": "naflex_processor.NaFlexProcessor"
|
| 9 |
+
},
|
| 10 |
+
"bos_token_id": 2,
|
| 11 |
+
"decoder_config": {
|
| 12 |
+
"_comment_params": "tied_embed=4.1M + 4x(self_attn=197K + cross_attn=229K + swiglu=786K + norms~2K)=4.85M",
|
| 13 |
+
"bos_token_id": 2,
|
| 14 |
+
"eos_token_id": 4,
|
| 15 |
+
"hidden_size": 256,
|
| 16 |
+
"intermediate_size": 1024,
|
| 17 |
+
"max_position_embeddings": 2048,
|
| 18 |
+
"num_attention_heads": 4,
|
| 19 |
+
"num_hidden_layers": 4,
|
| 20 |
+
"num_key_value_heads": 2,
|
| 21 |
+
"pad_token_id": 0,
|
| 22 |
+
"rms_norm_eps": 1e-06,
|
| 23 |
+
"rope_theta": 50000.0,
|
| 24 |
+
"tie_word_embeddings": true,
|
| 25 |
+
"vocab_size": 16000
|
| 26 |
+
},
|
| 27 |
+
"dtype": "bfloat16",
|
| 28 |
+
"eos_token_id": 4,
|
| 29 |
+
"model_type": "naflex_cross",
|
| 30 |
+
"pad_token_id": 5,
|
| 31 |
+
"tie_word_embeddings": true,
|
| 32 |
+
"transformers_version": "5.3.0",
|
| 33 |
+
"use_cache": false,
|
| 34 |
+
"vit_hidden_size": 384,
|
| 35 |
+
"vit_intermediate_size": 1536,
|
| 36 |
+
"vit_norm_eps": 1e-06,
|
| 37 |
+
"vit_num_heads": 6,
|
| 38 |
+
"vit_num_layers": 6,
|
| 39 |
+
"vit_patch_size": [
|
| 40 |
+
14,
|
| 41 |
+
7
|
| 42 |
+
],
|
| 43 |
+
"vit_rope_theta": 10000.0
|
| 44 |
+
}
|
generation_config.json
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 2,
|
| 4 |
+
"eos_token_id": [
|
| 5 |
+
4
|
| 6 |
+
],
|
| 7 |
+
"pad_token_id": 5,
|
| 8 |
+
"transformers_version": "5.3.0"
|
| 9 |
+
}
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9f57d9f3874799081a88ac91d4a73a91b9482c475216826f305e1fcd74704058
|
| 3 |
+
size 39378464
|
naflex_cross.py
ADDED
|
@@ -0,0 +1,643 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
NaFlex Cross-Attention VLM — HuggingFace-compatible.
|
| 3 |
+
|
| 4 |
+
Architecture:
|
| 5 |
+
- Encoder: NaFlex ViT — variable-resolution ViT with 2D factored RoPE
|
| 6 |
+
- Decoder: Causal LM (4-6 layers) with per-layer cross-attention to ViT features
|
| 7 |
+
|
| 8 |
+
NaFlex key property: patches carry (row, col) positions so any resolution / aspect
|
| 9 |
+
ratio is encoded without positional interpolation. FA2 dispatches automatically via
|
| 10 |
+
F.scaled_dot_product_attention on CUDA with fp16/bf16 inputs.
|
| 11 |
+
|
| 12 |
+
Default tiny config: ~19.8M parameters total (ViT 10.9M + decoder 8.9M).
|
| 13 |
+
vit: hidden=384, layers=6, heads=6, intermediate=1536, patch=16
|
| 14 |
+
dec: hidden=256, layers=4, heads=4, kv_heads=2, intermediate=1024, vocab=16000
|
| 15 |
+
|
| 16 |
+
Usage:
|
| 17 |
+
from rlatex.models.naflex_cross import NaFlexCrossConfig, NaFlexCrossForConditionalGeneration
|
| 18 |
+
NaFlexCrossConfig.register_for_auto_class()
|
| 19 |
+
NaFlexCrossForConditionalGeneration.register_for_auto_class("AutoModelForImageTextToText")
|
| 20 |
+
|
| 21 |
+
Inputs (forward / generate):
|
| 22 |
+
pixel_values [B, N_max, 3*P*P] padded flattened patches (float)
|
| 23 |
+
patch_positions [B, N_max, 2] (row, col) in patch units (long)
|
| 24 |
+
patch_attention_mask [B, N_max] True for valid patches (bool/float)
|
| 25 |
+
input_ids [B, T]
|
| 26 |
+
labels [B, T] -100 for ignored positions
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
import torch
|
| 30 |
+
import torch.nn as nn
|
| 31 |
+
import torch.nn.functional as F
|
| 32 |
+
from transformers import PretrainedConfig, PreTrainedModel
|
| 33 |
+
from transformers.modeling_outputs import CausalLMOutputWithPast
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
# ── Config ─────────────────────────────────────────────────────────────────────
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class NaFlexCrossConfig(PretrainedConfig):
|
| 40 |
+
model_type = "naflex_cross"
|
| 41 |
+
|
| 42 |
+
def __init__(
|
| 43 |
+
self,
|
| 44 |
+
# NaFlex ViT encoder
|
| 45 |
+
vit_patch_size: int | tuple[int, int] = 16, # (patch_h, patch_w) or single int for square
|
| 46 |
+
vit_hidden_size: int = 384,
|
| 47 |
+
vit_num_layers: int = 6,
|
| 48 |
+
vit_num_heads: int = 6, # head_dim = 384 / 6 = 64
|
| 49 |
+
vit_intermediate_size: int = 1536,
|
| 50 |
+
vit_norm_eps: float = 1e-6,
|
| 51 |
+
vit_rope_theta: float = 10000.0,
|
| 52 |
+
# Decoder
|
| 53 |
+
decoder_config: dict | None = None,
|
| 54 |
+
**kwargs,
|
| 55 |
+
):
|
| 56 |
+
kwargs.setdefault("tie_word_embeddings", True)
|
| 57 |
+
super().__init__(**kwargs)
|
| 58 |
+
# Normalise to (patch_h, patch_w) tuple; stored as list for JSON serialisability
|
| 59 |
+
if isinstance(vit_patch_size, int):
|
| 60 |
+
vit_patch_size = (vit_patch_size, vit_patch_size)
|
| 61 |
+
self.vit_patch_size = list(vit_patch_size)
|
| 62 |
+
self.vit_hidden_size = vit_hidden_size
|
| 63 |
+
self.vit_num_layers = vit_num_layers
|
| 64 |
+
self.vit_num_heads = vit_num_heads
|
| 65 |
+
self.vit_intermediate_size = vit_intermediate_size
|
| 66 |
+
self.vit_norm_eps = vit_norm_eps
|
| 67 |
+
self.vit_rope_theta = vit_rope_theta
|
| 68 |
+
self.decoder_config = decoder_config or {
|
| 69 |
+
# ~8.9M (incl. shared embed with vocab=16000)
|
| 70 |
+
"hidden_size": 256,
|
| 71 |
+
"num_hidden_layers": 4,
|
| 72 |
+
"num_attention_heads": 4, # head_dim = 256 / 4 = 64
|
| 73 |
+
"num_key_value_heads": 2,
|
| 74 |
+
"intermediate_size": 1024,
|
| 75 |
+
"vocab_size": 16000,
|
| 76 |
+
"max_position_embeddings": 2048,
|
| 77 |
+
"rope_theta": 50000.0,
|
| 78 |
+
"rms_norm_eps": 1e-6,
|
| 79 |
+
"bos_token_id": 2,
|
| 80 |
+
"eos_token_id": 4,
|
| 81 |
+
"pad_token_id": 0,
|
| 82 |
+
"tie_word_embeddings": True,
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
@property
|
| 86 |
+
def vit_patch_h(self) -> int:
|
| 87 |
+
return self.vit_patch_size[0]
|
| 88 |
+
|
| 89 |
+
@property
|
| 90 |
+
def vit_patch_w(self) -> int:
|
| 91 |
+
return self.vit_patch_size[1]
|
| 92 |
+
|
| 93 |
+
@property
|
| 94 |
+
def hidden_size(self) -> int:
|
| 95 |
+
return self.decoder_config.get("hidden_size", 256)
|
| 96 |
+
|
| 97 |
+
@property
|
| 98 |
+
def text_config(self) -> "NaFlexCrossConfig":
|
| 99 |
+
"""Compatibility shim — lets code like config.text_config.hidden_size work."""
|
| 100 |
+
return self
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
# ── Shared primitives ──────────────────────────────────────────────────────────
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
class RMSNorm(nn.Module):
|
| 107 |
+
def __init__(self, hidden_size: int, eps: float = 1e-6):
|
| 108 |
+
super().__init__()
|
| 109 |
+
self.weight = nn.Parameter(torch.ones(hidden_size))
|
| 110 |
+
self.variance_epsilon = eps
|
| 111 |
+
|
| 112 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 113 |
+
variance = x.pow(2).mean(-1, keepdim=True)
|
| 114 |
+
x = x * torch.rsqrt(variance + self.variance_epsilon)
|
| 115 |
+
return self.weight * x
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def _rotate_half_split(x: torch.Tensor) -> torch.Tensor:
|
| 119 |
+
"""Rotate each half of x independently: cat(-x2, x1) within each chunk."""
|
| 120 |
+
x1, x2 = x.chunk(2, dim=-1)
|
| 121 |
+
return torch.cat((-x2, x1), dim=-1)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
# ── 2D Factored RoPE (for ViT) ───────────────────────────────────────��─────────
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
class RotaryEmbedding2D(nn.Module):
|
| 128 |
+
"""
|
| 129 |
+
Factored 2D RoPE for NaFlex ViT.
|
| 130 |
+
|
| 131 |
+
head_dim is split in two equal halves:
|
| 132 |
+
first head_dim//2 dimensions carry row-axis RoPE
|
| 133 |
+
second head_dim//2 dimensions carry col-axis RoPE
|
| 134 |
+
No interpolation — position indices are raw patch row/col integers.
|
| 135 |
+
"""
|
| 136 |
+
|
| 137 |
+
def __init__(self, head_dim: int, theta: float = 10000.0):
|
| 138 |
+
super().__init__()
|
| 139 |
+
assert head_dim % 4 == 0, "head_dim must be divisible by 4 for 2D RoPE"
|
| 140 |
+
freq_dim = head_dim // 4
|
| 141 |
+
inv_freq = 1.0 / (
|
| 142 |
+
theta ** (torch.arange(0, freq_dim * 2, 2, dtype=torch.float32) / (freq_dim * 2))
|
| 143 |
+
)
|
| 144 |
+
self.register_buffer("inv_freq", inv_freq, persistent=False) # [freq_dim]
|
| 145 |
+
|
| 146 |
+
def forward(
|
| 147 |
+
self, positions: torch.Tensor # [B, N, 2] long: (row, col)
|
| 148 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 149 |
+
"""Returns cos, sin each of shape [B, N, head_dim]."""
|
| 150 |
+
rows = positions[..., 0].float() # [B, N]
|
| 151 |
+
cols = positions[..., 1].float() # [B, N]
|
| 152 |
+
|
| 153 |
+
row_freqs = torch.einsum("bn,d->bnd", rows, self.inv_freq) # [B, N, freq_dim]
|
| 154 |
+
col_freqs = torch.einsum("bn,d->bnd", cols, self.inv_freq) # [B, N, freq_dim]
|
| 155 |
+
|
| 156 |
+
row_emb = torch.cat([row_freqs, row_freqs], dim=-1) # [B, N, head_dim//2]
|
| 157 |
+
col_emb = torch.cat([col_freqs, col_freqs], dim=-1) # [B, N, head_dim//2]
|
| 158 |
+
emb = torch.cat([row_emb, col_emb], dim=-1) # [B, N, head_dim]
|
| 159 |
+
return emb.cos(), emb.sin()
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def _apply_rotary_2d(
|
| 163 |
+
q: torch.Tensor, # [B, H, N, head_dim]
|
| 164 |
+
k: torch.Tensor, # [B, H, N, head_dim]
|
| 165 |
+
cos: torch.Tensor, # [B, N, head_dim]
|
| 166 |
+
sin: torch.Tensor, # [B, N, head_dim]
|
| 167 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 168 |
+
"""Apply 2D factored RoPE: row-half and col-half are rotated separately."""
|
| 169 |
+
half = q.shape[-1] // 2
|
| 170 |
+
cos = cos.unsqueeze(1).to(q.dtype) # [B, 1, N, head_dim]
|
| 171 |
+
sin = sin.unsqueeze(1).to(q.dtype)
|
| 172 |
+
|
| 173 |
+
cos_row, cos_col = cos[..., :half], cos[..., half:]
|
| 174 |
+
sin_row, sin_col = sin[..., :half], sin[..., half:]
|
| 175 |
+
|
| 176 |
+
q_row, q_col = q[..., :half], q[..., half:]
|
| 177 |
+
k_row, k_col = k[..., :half], k[..., half:]
|
| 178 |
+
|
| 179 |
+
q_row = q_row * cos_row + _rotate_half_split(q_row) * sin_row
|
| 180 |
+
q_col = q_col * cos_col + _rotate_half_split(q_col) * sin_col
|
| 181 |
+
k_row = k_row * cos_row + _rotate_half_split(k_row) * sin_row
|
| 182 |
+
k_col = k_col * cos_col + _rotate_half_split(k_col) * sin_col
|
| 183 |
+
|
| 184 |
+
return torch.cat([q_row, q_col], dim=-1), torch.cat([k_row, k_col], dim=-1)
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
# ── 1D RoPE (for decoder) ─────────────────────────────────────────────────────
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
class RotaryEmbedding1D(nn.Module):
|
| 191 |
+
def __init__(self, dim: int, max_seq_len: int = 2048, theta: float = 50000.0):
|
| 192 |
+
super().__init__()
|
| 193 |
+
inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
|
| 194 |
+
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
| 195 |
+
|
| 196 |
+
def forward(self, seq_len: int, device: torch.device) -> tuple[torch.Tensor, torch.Tensor]:
|
| 197 |
+
t = torch.arange(seq_len, device=device, dtype=self.inv_freq.dtype)
|
| 198 |
+
freqs = torch.outer(t, self.inv_freq)
|
| 199 |
+
emb = torch.cat([freqs, freqs], dim=-1) # [T, dim]
|
| 200 |
+
return emb.cos(), emb.sin()
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def _apply_rotary_1d(
|
| 204 |
+
q: torch.Tensor,
|
| 205 |
+
k: torch.Tensor,
|
| 206 |
+
cos: torch.Tensor, # [1, 1, T, head_dim]
|
| 207 |
+
sin: torch.Tensor,
|
| 208 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 209 |
+
cos = cos.to(q.dtype)
|
| 210 |
+
sin = sin.to(q.dtype)
|
| 211 |
+
q = q * cos + _rotate_half_split(q) * sin
|
| 212 |
+
k = k * cos + _rotate_half_split(k) * sin
|
| 213 |
+
return q, k
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
# ── SwiGLU MLP ────────────────────────────────────────────────────────────────
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
class SwiGLU(nn.Module):
|
| 220 |
+
def __init__(self, hidden_size: int, intermediate_size: int):
|
| 221 |
+
super().__init__()
|
| 222 |
+
self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
|
| 223 |
+
self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
|
| 224 |
+
self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
|
| 225 |
+
|
| 226 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 227 |
+
return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
# ── NaFlex ViT ────────────────────────────────────────────────────────────────
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
class NaFlexViTAttention(nn.Module):
|
| 234 |
+
"""Full (non-causal) multi-head self-attention with 2D RoPE."""
|
| 235 |
+
|
| 236 |
+
def __init__(self, hidden_size: int, num_heads: int):
|
| 237 |
+
super().__init__()
|
| 238 |
+
self.num_heads = num_heads
|
| 239 |
+
self.head_dim = hidden_size // num_heads
|
| 240 |
+
self.scale = self.head_dim ** -0.5
|
| 241 |
+
self.q_proj = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 242 |
+
self.k_proj = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 243 |
+
self.v_proj = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 244 |
+
self.o_proj = nn.Linear(hidden_size, hidden_size, bias=False)
|
| 245 |
+
|
| 246 |
+
def forward(
|
| 247 |
+
self,
|
| 248 |
+
x: torch.Tensor, # [B, N, D]
|
| 249 |
+
cos: torch.Tensor, # [B, N, head_dim]
|
| 250 |
+
sin: torch.Tensor,
|
| 251 |
+
attention_mask: torch.Tensor | None = None, # [B, 1, 1, N] additive
|
| 252 |
+
) -> torch.Tensor:
|
| 253 |
+
B, N, _ = x.shape
|
| 254 |
+
q = self.q_proj(x).view(B, N, self.num_heads, self.head_dim).transpose(1, 2)
|
| 255 |
+
k = self.k_proj(x).view(B, N, self.num_heads, self.head_dim).transpose(1, 2)
|
| 256 |
+
v = self.v_proj(x).view(B, N, self.num_heads, self.head_dim).transpose(1, 2)
|
| 257 |
+
|
| 258 |
+
q, k = _apply_rotary_2d(q, k, cos, sin)
|
| 259 |
+
|
| 260 |
+
out = F.scaled_dot_product_attention(
|
| 261 |
+
q, k, v,
|
| 262 |
+
attn_mask=attention_mask,
|
| 263 |
+
is_causal=False,
|
| 264 |
+
scale=self.scale,
|
| 265 |
+
)
|
| 266 |
+
return self.o_proj(out.transpose(1, 2).reshape(B, N, -1))
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
class NaFlexViTLayer(nn.Module):
|
| 270 |
+
def __init__(self, hidden_size: int, num_heads: int, intermediate_size: int, norm_eps: float):
|
| 271 |
+
super().__init__()
|
| 272 |
+
self.norm1 = RMSNorm(hidden_size, eps=norm_eps)
|
| 273 |
+
self.attn = NaFlexViTAttention(hidden_size, num_heads)
|
| 274 |
+
self.norm2 = RMSNorm(hidden_size, eps=norm_eps)
|
| 275 |
+
self.mlp = nn.Sequential(
|
| 276 |
+
nn.Linear(hidden_size, intermediate_size, bias=False),
|
| 277 |
+
nn.GELU(),
|
| 278 |
+
nn.Linear(intermediate_size, hidden_size, bias=False),
|
| 279 |
+
)
|
| 280 |
+
|
| 281 |
+
def forward(
|
| 282 |
+
self,
|
| 283 |
+
x: torch.Tensor,
|
| 284 |
+
cos: torch.Tensor,
|
| 285 |
+
sin: torch.Tensor,
|
| 286 |
+
attention_mask: torch.Tensor | None = None,
|
| 287 |
+
) -> torch.Tensor:
|
| 288 |
+
x = x + self.attn(self.norm1(x), cos, sin, attention_mask)
|
| 289 |
+
x = x + self.mlp(self.norm2(x))
|
| 290 |
+
return x
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
class _ParamGroup:
|
| 294 |
+
"""
|
| 295 |
+
Lightweight non-Module grouping of parameters for freeze / LR partitioning.
|
| 296 |
+
train_sft.py resolves sub-modules by name and expects a .parameters() iterable.
|
| 297 |
+
Using a plain object avoids double-registering weights with PyTorch.
|
| 298 |
+
"""
|
| 299 |
+
def __init__(self, *modules: nn.Module):
|
| 300 |
+
self._modules = modules
|
| 301 |
+
|
| 302 |
+
def parameters(self):
|
| 303 |
+
seen: set[int] = set()
|
| 304 |
+
for m in self._modules:
|
| 305 |
+
for p in m.parameters():
|
| 306 |
+
if id(p) not in seen:
|
| 307 |
+
seen.add(id(p))
|
| 308 |
+
yield p
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
class NaFlexViTEncoder(nn.Module):
|
| 312 |
+
"""
|
| 313 |
+
NaFlex vision transformer encoder.
|
| 314 |
+
|
| 315 |
+
Accepts variable-resolution images as padded batches of flattened patches.
|
| 316 |
+
Each patch carries its (row, col) grid position for 2D RoPE — no positional
|
| 317 |
+
interpolation needed for arbitrary resolutions.
|
| 318 |
+
"""
|
| 319 |
+
|
| 320 |
+
def __init__(self, config: NaFlexCrossConfig):
|
| 321 |
+
super().__init__()
|
| 322 |
+
H = config.vit_hidden_size
|
| 323 |
+
self.patch_embed = nn.Linear(3 * config.vit_patch_h * config.vit_patch_w, H, bias=False)
|
| 324 |
+
self.layers = nn.ModuleList([
|
| 325 |
+
NaFlexViTLayer(
|
| 326 |
+
H,
|
| 327 |
+
config.vit_num_heads,
|
| 328 |
+
config.vit_intermediate_size,
|
| 329 |
+
config.vit_norm_eps,
|
| 330 |
+
)
|
| 331 |
+
for _ in range(config.vit_num_layers)
|
| 332 |
+
])
|
| 333 |
+
self.norm = RMSNorm(H, eps=config.vit_norm_eps)
|
| 334 |
+
self.rope = RotaryEmbedding2D(
|
| 335 |
+
head_dim=H // config.vit_num_heads,
|
| 336 |
+
theta=config.vit_rope_theta,
|
| 337 |
+
)
|
| 338 |
+
|
| 339 |
+
@property
|
| 340 |
+
def blocks(self) -> nn.ModuleList:
|
| 341 |
+
"""Alias for train_sft.py which resolves visual layers as `.blocks`."""
|
| 342 |
+
return self.layers
|
| 343 |
+
|
| 344 |
+
def forward(
|
| 345 |
+
self,
|
| 346 |
+
pixel_values: torch.Tensor, # [B, N, 3*P*P]
|
| 347 |
+
patch_positions: torch.Tensor, # [B, N, 2] long: (row, col)
|
| 348 |
+
patch_attention_mask: torch.Tensor | None = None, # [B, N] bool/float
|
| 349 |
+
) -> torch.Tensor: # [B, N, H]
|
| 350 |
+
x = self.patch_embed(pixel_values)
|
| 351 |
+
cos, sin = self.rope(patch_positions)
|
| 352 |
+
|
| 353 |
+
# Additive mask: valid positions get 0, padding positions get -inf
|
| 354 |
+
attn_mask = None
|
| 355 |
+
if patch_attention_mask is not None:
|
| 356 |
+
valid = patch_attention_mask.float() # [B, N]
|
| 357 |
+
fill = torch.finfo(x.dtype).min
|
| 358 |
+
attn_mask = (1.0 - valid.unsqueeze(1).unsqueeze(2)) * fill # [B, 1, 1, N]
|
| 359 |
+
|
| 360 |
+
for layer in self.layers:
|
| 361 |
+
x = layer(x, cos, sin, attn_mask)
|
| 362 |
+
|
| 363 |
+
return self.norm(x)
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
# ── Decoder ───────────────────────────────────────────────────────────────────
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
class GQAttention(nn.Module):
|
| 370 |
+
"""Grouped-query attention for decoder self- and cross-attention."""
|
| 371 |
+
|
| 372 |
+
def __init__(
|
| 373 |
+
self,
|
| 374 |
+
hidden_size: int,
|
| 375 |
+
num_heads: int,
|
| 376 |
+
num_kv_heads: int,
|
| 377 |
+
kv_input_size: int | None = None,
|
| 378 |
+
is_causal: bool = True,
|
| 379 |
+
):
|
| 380 |
+
super().__init__()
|
| 381 |
+
self.num_heads = num_heads
|
| 382 |
+
self.num_kv_heads = num_kv_heads
|
| 383 |
+
self.num_kv_groups = num_heads // num_kv_heads
|
| 384 |
+
self.head_dim = hidden_size // num_heads
|
| 385 |
+
self.scale = self.head_dim ** -0.5
|
| 386 |
+
self.is_causal = is_causal
|
| 387 |
+
|
| 388 |
+
kv_size = kv_input_size if kv_input_size is not None else hidden_size
|
| 389 |
+
self.q_proj = nn.Linear(hidden_size, num_heads * self.head_dim, bias=False)
|
| 390 |
+
self.k_proj = nn.Linear(kv_size, num_kv_heads * self.head_dim, bias=False)
|
| 391 |
+
self.v_proj = nn.Linear(kv_size, num_kv_heads * self.head_dim, bias=False)
|
| 392 |
+
self.o_proj = nn.Linear(num_heads * self.head_dim, hidden_size, bias=False)
|
| 393 |
+
|
| 394 |
+
def forward(
|
| 395 |
+
self,
|
| 396 |
+
hidden_states: torch.Tensor, # [B, T, D]
|
| 397 |
+
kv_source: torch.Tensor | None = None, # [B, S, D_kv] — None → self-attn
|
| 398 |
+
attention_mask: torch.Tensor | None = None,
|
| 399 |
+
cos: torch.Tensor | None = None, # [T, head_dim] — 1D RoPE for self-attn
|
| 400 |
+
sin: torch.Tensor | None = None,
|
| 401 |
+
) -> torch.Tensor:
|
| 402 |
+
B, T, _ = hidden_states.shape
|
| 403 |
+
src = hidden_states if kv_source is None else kv_source
|
| 404 |
+
S = src.shape[1]
|
| 405 |
+
|
| 406 |
+
q = self.q_proj(hidden_states).view(B, T, self.num_heads, self.head_dim).transpose(1, 2)
|
| 407 |
+
k = self.k_proj(src).view(B, S, self.num_kv_heads, self.head_dim).transpose(1, 2)
|
| 408 |
+
v = self.v_proj(src).view(B, S, self.num_kv_heads, self.head_dim).transpose(1, 2)
|
| 409 |
+
|
| 410 |
+
if cos is not None and kv_source is None:
|
| 411 |
+
cos_t = cos[:T].unsqueeze(0).unsqueeze(0) # [1, 1, T, head_dim]
|
| 412 |
+
sin_t = sin[:T].unsqueeze(0).unsqueeze(0)
|
| 413 |
+
q, k = _apply_rotary_1d(q, k, cos_t, sin_t)
|
| 414 |
+
|
| 415 |
+
if self.num_kv_groups > 1:
|
| 416 |
+
k = k.unsqueeze(2).expand(-1, -1, self.num_kv_groups, -1, -1).reshape(
|
| 417 |
+
B, self.num_heads, S, self.head_dim
|
| 418 |
+
)
|
| 419 |
+
v = v.unsqueeze(2).expand(-1, -1, self.num_kv_groups, -1, -1).reshape(
|
| 420 |
+
B, self.num_heads, S, self.head_dim
|
| 421 |
+
)
|
| 422 |
+
|
| 423 |
+
out = F.scaled_dot_product_attention(
|
| 424 |
+
q, k, v,
|
| 425 |
+
attn_mask=attention_mask,
|
| 426 |
+
is_causal=(self.is_causal and kv_source is None and attention_mask is None),
|
| 427 |
+
scale=self.scale,
|
| 428 |
+
)
|
| 429 |
+
return self.o_proj(out.transpose(1, 2).reshape(B, T, -1))
|
| 430 |
+
|
| 431 |
+
|
| 432 |
+
class NaFlexDecoderLayer(nn.Module):
|
| 433 |
+
def __init__(self, dcfg: dict, vit_hidden: int):
|
| 434 |
+
super().__init__()
|
| 435 |
+
H = dcfg["hidden_size"]
|
| 436 |
+
num_heads = dcfg["num_attention_heads"]
|
| 437 |
+
num_kv_heads = dcfg["num_key_value_heads"]
|
| 438 |
+
eps = dcfg["rms_norm_eps"]
|
| 439 |
+
|
| 440 |
+
self.norm1 = RMSNorm(H, eps=eps)
|
| 441 |
+
self.self_attn = GQAttention(H, num_heads, num_kv_heads, is_causal=True)
|
| 442 |
+
|
| 443 |
+
self.norm_cross = RMSNorm(H, eps=eps)
|
| 444 |
+
self.cross_attn = GQAttention(
|
| 445 |
+
H, num_heads, num_kv_heads, kv_input_size=vit_hidden, is_causal=False
|
| 446 |
+
)
|
| 447 |
+
|
| 448 |
+
self.norm2 = RMSNorm(H, eps=eps)
|
| 449 |
+
self.mlp = SwiGLU(H, dcfg["intermediate_size"])
|
| 450 |
+
|
| 451 |
+
def forward(
|
| 452 |
+
self,
|
| 453 |
+
x: torch.Tensor,
|
| 454 |
+
vision_feats: torch.Tensor,
|
| 455 |
+
vision_mask: torch.Tensor | None = None,
|
| 456 |
+
cos: torch.Tensor | None = None,
|
| 457 |
+
sin: torch.Tensor | None = None,
|
| 458 |
+
) -> torch.Tensor:
|
| 459 |
+
x = x + self.self_attn(self.norm1(x), cos=cos, sin=sin)
|
| 460 |
+
x = x + self.cross_attn(self.norm_cross(x), kv_source=vision_feats, attention_mask=vision_mask)
|
| 461 |
+
x = x + self.mlp(self.norm2(x))
|
| 462 |
+
return x
|
| 463 |
+
|
| 464 |
+
|
| 465 |
+
# ── Full model ─────────────────────────────────────────────────────────────────
|
| 466 |
+
|
| 467 |
+
|
| 468 |
+
class NaFlexCrossForConditionalGeneration(PreTrainedModel):
|
| 469 |
+
"""
|
| 470 |
+
NaFlex VLM: variable-resolution ViT encoder + causal cross-attention decoder.
|
| 471 |
+
|
| 472 |
+
Compatible with HuggingFace Trainer / SFTTrainer.
|
| 473 |
+
FA2 is used automatically by F.scaled_dot_product_attention when inputs are
|
| 474 |
+
fp16/bf16 on CUDA (no extra flags needed).
|
| 475 |
+
Gradient checkpointing is enabled with model.gradient_checkpointing_enable().
|
| 476 |
+
"""
|
| 477 |
+
|
| 478 |
+
config_class = NaFlexCrossConfig
|
| 479 |
+
main_input_name = "input_ids"
|
| 480 |
+
supports_gradient_checkpointing = True
|
| 481 |
+
_supports_flash_attn_2 = True
|
| 482 |
+
_tied_weights_keys = {"lm_head.weight": "embed_tokens.weight"}
|
| 483 |
+
|
| 484 |
+
def __init__(self, config: NaFlexCrossConfig):
|
| 485 |
+
super().__init__(config)
|
| 486 |
+
dcfg = config.decoder_config
|
| 487 |
+
|
| 488 |
+
# Vision encoder
|
| 489 |
+
self.visual = NaFlexViTEncoder(config)
|
| 490 |
+
|
| 491 |
+
# Decoder
|
| 492 |
+
H = dcfg["hidden_size"]
|
| 493 |
+
vocab = dcfg["vocab_size"]
|
| 494 |
+
pad_id = dcfg.get("pad_token_id", 0)
|
| 495 |
+
|
| 496 |
+
self.embed_tokens = nn.Embedding(vocab, H, padding_idx=pad_id)
|
| 497 |
+
self.layers = nn.ModuleList([
|
| 498 |
+
NaFlexDecoderLayer(dcfg, vit_hidden=config.vit_hidden_size)
|
| 499 |
+
for _ in range(dcfg["num_hidden_layers"])
|
| 500 |
+
])
|
| 501 |
+
self.norm = RMSNorm(H, eps=dcfg["rms_norm_eps"])
|
| 502 |
+
self.lm_head = nn.Linear(H, vocab, bias=False)
|
| 503 |
+
|
| 504 |
+
if dcfg.get("tie_word_embeddings", True):
|
| 505 |
+
self.lm_head.weight = self.embed_tokens.weight
|
| 506 |
+
|
| 507 |
+
self.rotary_emb = RotaryEmbedding1D(
|
| 508 |
+
dim=H // dcfg["num_attention_heads"],
|
| 509 |
+
max_seq_len=dcfg["max_position_embeddings"],
|
| 510 |
+
theta=dcfg.get("rope_theta", 50000.0),
|
| 511 |
+
)
|
| 512 |
+
|
| 513 |
+
self.gradient_checkpointing = False
|
| 514 |
+
self.post_init()
|
| 515 |
+
|
| 516 |
+
def get_input_embeddings(self) -> nn.Embedding:
|
| 517 |
+
return self.embed_tokens
|
| 518 |
+
|
| 519 |
+
def set_input_embeddings(self, value: nn.Embedding):
|
| 520 |
+
self.embed_tokens = value
|
| 521 |
+
|
| 522 |
+
def get_output_embeddings(self) -> nn.Linear:
|
| 523 |
+
return self.lm_head
|
| 524 |
+
|
| 525 |
+
def set_output_embeddings(self, value: nn.Linear):
|
| 526 |
+
self.lm_head = value
|
| 527 |
+
|
| 528 |
+
@property
|
| 529 |
+
def language_model(self) -> _ParamGroup:
|
| 530 |
+
"""
|
| 531 |
+
Pseudo-module grouping the decoder components.
|
| 532 |
+
Lets train_sft.py resolve the LM side for freeze / per-component LR /
|
| 533 |
+
gradient norm logging without double-registering weights.
|
| 534 |
+
"""
|
| 535 |
+
return _ParamGroup(self.embed_tokens, self.layers, self.norm, self.lm_head)
|
| 536 |
+
|
| 537 |
+
def tie_weights(self, **kwargs):
|
| 538 |
+
if self.config.tie_word_embeddings:
|
| 539 |
+
self.lm_head.weight = self.embed_tokens.weight
|
| 540 |
+
|
| 541 |
+
def _encode_images(
|
| 542 |
+
self,
|
| 543 |
+
pixel_values: torch.Tensor,
|
| 544 |
+
patch_positions: torch.Tensor,
|
| 545 |
+
patch_attention_mask: torch.Tensor | None,
|
| 546 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 547 |
+
"""Encode images and return (vision_feats [B,N,D], vision_mask [B,1,1,N])."""
|
| 548 |
+
vision_feats = self.visual(pixel_values, patch_positions, patch_attention_mask)
|
| 549 |
+
|
| 550 |
+
vision_mask = None
|
| 551 |
+
if patch_attention_mask is not None:
|
| 552 |
+
fill = torch.finfo(vision_feats.dtype).min
|
| 553 |
+
valid = patch_attention_mask.float()
|
| 554 |
+
vision_mask = (1.0 - valid.unsqueeze(1).unsqueeze(2)) * fill # [B, 1, 1, N]
|
| 555 |
+
|
| 556 |
+
return vision_feats, vision_mask
|
| 557 |
+
|
| 558 |
+
def forward(
|
| 559 |
+
self,
|
| 560 |
+
input_ids: torch.Tensor,
|
| 561 |
+
pixel_values: torch.Tensor,
|
| 562 |
+
patch_positions: torch.Tensor,
|
| 563 |
+
patch_attention_mask: torch.Tensor | None = None,
|
| 564 |
+
attention_mask: torch.Tensor | None = None,
|
| 565 |
+
labels: torch.Tensor | None = None,
|
| 566 |
+
# Precomputed vision features (fast generation: encode once, decode many)
|
| 567 |
+
vision_feats: torch.Tensor | None = None,
|
| 568 |
+
vision_mask: torch.Tensor | None = None,
|
| 569 |
+
**kwargs,
|
| 570 |
+
) -> CausalLMOutputWithPast:
|
| 571 |
+
if vision_feats is None:
|
| 572 |
+
# Cast pixel_values to model dtype (processor always returns float32)
|
| 573 |
+
pixel_values = pixel_values.to(self.visual.patch_embed.weight.dtype)
|
| 574 |
+
vision_feats, vision_mask = self._encode_images(
|
| 575 |
+
pixel_values, patch_positions, patch_attention_mask
|
| 576 |
+
)
|
| 577 |
+
|
| 578 |
+
x = self.embed_tokens(input_ids)
|
| 579 |
+
cos, sin = self.rotary_emb(input_ids.shape[1], device=x.device)
|
| 580 |
+
|
| 581 |
+
for layer in self.layers:
|
| 582 |
+
if self.gradient_checkpointing and self.training:
|
| 583 |
+
x = torch.utils.checkpoint.checkpoint(
|
| 584 |
+
layer, x, vision_feats, vision_mask, cos, sin, use_reentrant=False
|
| 585 |
+
)
|
| 586 |
+
else:
|
| 587 |
+
x = layer(x, vision_feats=vision_feats, vision_mask=vision_mask, cos=cos, sin=sin)
|
| 588 |
+
|
| 589 |
+
x = self.norm(x)
|
| 590 |
+
logits = self.lm_head(x)
|
| 591 |
+
|
| 592 |
+
loss = None
|
| 593 |
+
if labels is not None:
|
| 594 |
+
loss = F.cross_entropy(
|
| 595 |
+
logits[:, :-1].reshape(-1, logits.shape[-1]),
|
| 596 |
+
labels[:, 1:].reshape(-1),
|
| 597 |
+
ignore_index=-100,
|
| 598 |
+
)
|
| 599 |
+
|
| 600 |
+
return CausalLMOutputWithPast(loss=loss, logits=logits)
|
| 601 |
+
|
| 602 |
+
@torch.inference_mode()
|
| 603 |
+
def generate(
|
| 604 |
+
self,
|
| 605 |
+
pixel_values: torch.Tensor,
|
| 606 |
+
patch_positions: torch.Tensor,
|
| 607 |
+
patch_attention_mask: torch.Tensor | None = None,
|
| 608 |
+
input_ids: torch.Tensor | None = None, # optional text prompt
|
| 609 |
+
max_new_tokens: int = 256,
|
| 610 |
+
bos_token_id: int | None = None,
|
| 611 |
+
eos_token_id: int | None = None,
|
| 612 |
+
**kwargs,
|
| 613 |
+
) -> torch.Tensor:
|
| 614 |
+
"""
|
| 615 |
+
Greedy decode. Returns the full sequence (prompt + generated tokens) so
|
| 616 |
+
that callers can slice off the prompt with ``output[:, input_ids.shape[1]:]``.
|
| 617 |
+
If ``input_ids`` is not provided, generation starts from BOS.
|
| 618 |
+
"""
|
| 619 |
+
dcfg = self.config.decoder_config
|
| 620 |
+
bos = bos_token_id or dcfg.get("bos_token_id", 2)
|
| 621 |
+
eos = eos_token_id or dcfg.get("eos_token_id", 4)
|
| 622 |
+
|
| 623 |
+
pixel_values = pixel_values.to(self.visual.patch_embed.weight.dtype)
|
| 624 |
+
vision_feats, vision_mask = self._encode_images(
|
| 625 |
+
pixel_values, patch_positions, patch_attention_mask
|
| 626 |
+
)
|
| 627 |
+
|
| 628 |
+
if input_ids is not None:
|
| 629 |
+
ids = input_ids
|
| 630 |
+
else:
|
| 631 |
+
ids = pixel_values.new_full((pixel_values.shape[0], 1), bos, dtype=torch.long)
|
| 632 |
+
|
| 633 |
+
for _ in range(max_new_tokens):
|
| 634 |
+
out = self.forward(
|
| 635 |
+
ids, pixel_values, patch_positions, patch_attention_mask,
|
| 636 |
+
vision_feats=vision_feats, vision_mask=vision_mask,
|
| 637 |
+
)
|
| 638 |
+
next_tok = out.logits[:, -1].argmax(dim=-1, keepdim=True)
|
| 639 |
+
ids = torch.cat([ids, next_tok], dim=1)
|
| 640 |
+
if (next_tok == eos).all():
|
| 641 |
+
break
|
| 642 |
+
|
| 643 |
+
return ids # full sequence (prompt + generated)
|
naflex_processor.py
ADDED
|
@@ -0,0 +1,342 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
NaFlexProcessor — image + text processor for NaFlexCrossForConditionalGeneration.
|
| 3 |
+
|
| 4 |
+
Image pipeline:
|
| 5 |
+
1. Convert to RGB
|
| 6 |
+
2. Resize so the patch count stays within `max_num_patches`, snapping H and W
|
| 7 |
+
to multiples of patch_h / patch_w respectively (preserves aspect ratio).
|
| 8 |
+
3. Normalise with per-channel mean/std.
|
| 9 |
+
4. Extract patches row-major; record (row, col) integer position of each patch.
|
| 10 |
+
5. Pad across the batch to the longest sequence; return patch_attention_mask.
|
| 11 |
+
|
| 12 |
+
Text pipeline:
|
| 13 |
+
Standard HuggingFace tokenizer with padding / truncation.
|
| 14 |
+
|
| 15 |
+
Usage:
|
| 16 |
+
processor = NaFlexProcessor.from_pretrained("checkpoints/qwen2vl-24m",
|
| 17 |
+
patch_size=(16, 16))
|
| 18 |
+
batch = processor(
|
| 19 |
+
text=texts,
|
| 20 |
+
images=image_inputs,
|
| 21 |
+
return_tensors="pt",
|
| 22 |
+
padding="longest",
|
| 23 |
+
truncation=True,
|
| 24 |
+
max_length=512,
|
| 25 |
+
)
|
| 26 |
+
# batch keys: input_ids, attention_mask,
|
| 27 |
+
# pixel_values, patch_positions, patch_attention_mask
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
import math
|
| 31 |
+
from typing import List, Optional, Union
|
| 32 |
+
|
| 33 |
+
import numpy as np
|
| 34 |
+
import torch
|
| 35 |
+
from PIL import Image
|
| 36 |
+
from transformers import AutoTokenizer, BatchFeature, ProcessorMixin
|
| 37 |
+
from transformers.tokenization_utils_base import PaddingStrategy, TruncationStrategy
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
_DEFAULT_MEAN = (0.7931, 0.7931, 0.7931)
|
| 41 |
+
_DEFAULT_STD = (0.1738, 0.1738, 0.1738)
|
| 42 |
+
|
| 43 |
+
# Cross-attention template: images are encoded by the ViT, NOT injected as tokens.
|
| 44 |
+
# Image content blocks are silently dropped; only text reaches the tokenizer.
|
| 45 |
+
_DEFAULT_CHAT_TEMPLATE = (
|
| 46 |
+
"{%- for message in messages %}\n"
|
| 47 |
+
"{{- '<|im_start|>' }\n"
|
| 48 |
+
"{%- if message['content'] is string %}\n"
|
| 49 |
+
"{{- message['content'] }}\n"
|
| 50 |
+
"{%- else %}\n"
|
| 51 |
+
"{%- for content in message['content'] %}\n"
|
| 52 |
+
"{%- if content['type'] == 'text' %}\n"
|
| 53 |
+
"{{- content['text'] }}\n"
|
| 54 |
+
"{%- endif %}\n"
|
| 55 |
+
"{%- endfor %}\n"
|
| 56 |
+
"{%- endif %}\n"
|
| 57 |
+
"{{- '<|im_end|>\\n' }}\n"
|
| 58 |
+
"{%- endfor %}\n"
|
| 59 |
+
"{%- if add_generation_prompt %}\n"
|
| 60 |
+
"{{- '<|im_start|>' }}\n"
|
| 61 |
+
"{%- endif %}\n"
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
class NaFlexProcessor(ProcessorMixin):
|
| 66 |
+
"""
|
| 67 |
+
Processor for NaFlexCrossForConditionalGeneration.
|
| 68 |
+
|
| 69 |
+
Args:
|
| 70 |
+
tokenizer: Any HuggingFace tokenizer.
|
| 71 |
+
patch_size: (patch_h, patch_w) or single int for square patches.
|
| 72 |
+
max_num_patches: Maximum patches per image (controls resolution budget).
|
| 73 |
+
image_mean: Per-channel normalisation mean (C,).
|
| 74 |
+
image_std: Per-channel normalisation std (C,).
|
| 75 |
+
"""
|
| 76 |
+
|
| 77 |
+
attributes = ["tokenizer"]
|
| 78 |
+
tokenizer_class = "AutoTokenizer"
|
| 79 |
+
# Sentinel: NaFlex has no image tokens in the text sequence.
|
| 80 |
+
# train_sft.py uses this to mask image tokens in labels; -1 is never a real token ID.
|
| 81 |
+
image_token_id: int = -1
|
| 82 |
+
|
| 83 |
+
def __init__(
|
| 84 |
+
self,
|
| 85 |
+
tokenizer,
|
| 86 |
+
patch_size: Union[int, tuple] = 16,
|
| 87 |
+
max_num_patches: int = 1024,
|
| 88 |
+
image_mean: tuple = _DEFAULT_MEAN,
|
| 89 |
+
image_std: tuple = _DEFAULT_STD,
|
| 90 |
+
chat_template: Optional[str] = None,
|
| 91 |
+
):
|
| 92 |
+
if isinstance(patch_size, int):
|
| 93 |
+
patch_size = (patch_size, patch_size)
|
| 94 |
+
self.patch_size = list(patch_size) # serialised as [patch_h, patch_w]
|
| 95 |
+
self.patch_h, self.patch_w = patch_size
|
| 96 |
+
self.max_num_patches = max_num_patches
|
| 97 |
+
self.image_mean = np.array(image_mean, dtype=np.float32)
|
| 98 |
+
self.image_std = np.array(image_std, dtype=np.float32)
|
| 99 |
+
# ProcessorMixin stores chat_template as its own attribute via kwargs
|
| 100 |
+
super().__init__(tokenizer, chat_template=chat_template or _DEFAULT_CHAT_TEMPLATE)
|
| 101 |
+
|
| 102 |
+
# ── Save / load ───────────────────────────────────────────────────────────
|
| 103 |
+
|
| 104 |
+
def save_pretrained(self, save_directory: str, **kwargs):
|
| 105 |
+
"""Save processor config, tokenizer, chat template, and a copy of this module."""
|
| 106 |
+
import json, shutil, os
|
| 107 |
+
|
| 108 |
+
# Always enforce the NaFlex cross-attention template (not the early-fusion one
|
| 109 |
+
# that may have been inherited from a Qwen2VL tokenizer source)
|
| 110 |
+
if self.tokenizer.chat_template != _DEFAULT_CHAT_TEMPLATE:
|
| 111 |
+
self.tokenizer.chat_template = _DEFAULT_CHAT_TEMPLATE
|
| 112 |
+
|
| 113 |
+
super().save_pretrained(save_directory, **kwargs)
|
| 114 |
+
|
| 115 |
+
# Write chat_template.jinja and also stamp it directly into tokenizer_config.json.
|
| 116 |
+
# tokenizer.save_pretrained() only writes chat_template there if the tokenizer
|
| 117 |
+
# was originally constructed with it, so we patch it ourselves.
|
| 118 |
+
with open(os.path.join(save_directory, "chat_template.jinja"), "w") as f:
|
| 119 |
+
f.write(self.tokenizer.chat_template)
|
| 120 |
+
tok_cfg_path = os.path.join(save_directory, "tokenizer_config.json")
|
| 121 |
+
if os.path.isfile(tok_cfg_path):
|
| 122 |
+
with open(tok_cfg_path) as f:
|
| 123 |
+
tok_cfg = json.load(f)
|
| 124 |
+
tok_cfg["chat_template"] = self.tokenizer.chat_template
|
| 125 |
+
with open(tok_cfg_path, "w") as f:
|
| 126 |
+
json.dump(tok_cfg, f, indent=2)
|
| 127 |
+
|
| 128 |
+
# Overwrite processor_config.json with all image config fields
|
| 129 |
+
cfg_path = os.path.join(save_directory, "processor_config.json")
|
| 130 |
+
with open(cfg_path) as f:
|
| 131 |
+
cfg = json.load(f)
|
| 132 |
+
cfg["patch_size"] = self.patch_size
|
| 133 |
+
cfg["max_num_patches"] = self.max_num_patches
|
| 134 |
+
cfg["image_mean"] = self.image_mean.tolist()
|
| 135 |
+
cfg["image_std"] = self.image_std.tolist()
|
| 136 |
+
cfg["processor_class"] = "NaFlexProcessor"
|
| 137 |
+
with open(cfg_path, "w") as f:
|
| 138 |
+
json.dump(cfg, f, indent=2)
|
| 139 |
+
|
| 140 |
+
# Copy module file so AutoProcessor can load with trust_remote_code=True
|
| 141 |
+
src = os.path.join(os.path.dirname(__file__), "naflex_processor.py")
|
| 142 |
+
shutil.copy(src, os.path.join(save_directory, "naflex_processor.py"))
|
| 143 |
+
|
| 144 |
+
# Add AutoProcessor entry to config.json auto_map
|
| 145 |
+
main_cfg_path = os.path.join(save_directory, "config.json")
|
| 146 |
+
if os.path.exists(main_cfg_path):
|
| 147 |
+
with open(main_cfg_path) as f:
|
| 148 |
+
main_cfg = json.load(f)
|
| 149 |
+
main_cfg.setdefault("auto_map", {})
|
| 150 |
+
main_cfg["auto_map"]["AutoProcessor"] = "naflex_processor.NaFlexProcessor"
|
| 151 |
+
with open(main_cfg_path, "w") as f:
|
| 152 |
+
json.dump(main_cfg, f, indent=2)
|
| 153 |
+
|
| 154 |
+
@classmethod
|
| 155 |
+
def from_pretrained(
|
| 156 |
+
cls,
|
| 157 |
+
pretrained_model_name_or_path: str,
|
| 158 |
+
patch_size: Union[int, tuple, None] = None,
|
| 159 |
+
max_num_patches: Optional[int] = None,
|
| 160 |
+
image_mean: Optional[tuple] = None,
|
| 161 |
+
image_std: Optional[tuple] = None,
|
| 162 |
+
**kwargs,
|
| 163 |
+
) -> "NaFlexProcessor":
|
| 164 |
+
import json, os
|
| 165 |
+
# Strip kwargs that belong to AutoProcessor/tokenizer infrastructure
|
| 166 |
+
kwargs.pop("use_fast", None)
|
| 167 |
+
trust_remote_code = kwargs.pop("trust_remote_code", None)
|
| 168 |
+
|
| 169 |
+
# Read saved image config (if loading from a checkpoint directory)
|
| 170 |
+
saved = {}
|
| 171 |
+
cfg_path = os.path.join(pretrained_model_name_or_path, "processor_config.json")
|
| 172 |
+
if os.path.isfile(cfg_path):
|
| 173 |
+
with open(cfg_path) as f:
|
| 174 |
+
saved = json.load(f)
|
| 175 |
+
|
| 176 |
+
tokenizer = AutoTokenizer.from_pretrained(
|
| 177 |
+
pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
|
| 178 |
+
)
|
| 179 |
+
|
| 180 |
+
# Load chat template: checkpoint file takes priority, then built-in default.
|
| 181 |
+
# This overwrites any early-fusion template inherited from a Qwen2VL tokenizer.
|
| 182 |
+
template_path = os.path.join(pretrained_model_name_or_path, "chat_template.jinja")
|
| 183 |
+
if os.path.isfile(template_path):
|
| 184 |
+
with open(template_path) as f:
|
| 185 |
+
chat_template = f.read()
|
| 186 |
+
else:
|
| 187 |
+
chat_template = _DEFAULT_CHAT_TEMPLATE
|
| 188 |
+
|
| 189 |
+
# Fall back to vit_patch_size from model config.json if not in processor_config
|
| 190 |
+
if patch_size is None and "patch_size" not in saved:
|
| 191 |
+
model_cfg_path = os.path.join(pretrained_model_name_or_path, "config.json")
|
| 192 |
+
if os.path.isfile(model_cfg_path):
|
| 193 |
+
with open(model_cfg_path) as f:
|
| 194 |
+
model_cfg = json.load(f)
|
| 195 |
+
if "vit_patch_size" in model_cfg:
|
| 196 |
+
saved["patch_size"] = model_cfg["vit_patch_size"]
|
| 197 |
+
|
| 198 |
+
return cls(
|
| 199 |
+
tokenizer=tokenizer,
|
| 200 |
+
chat_template=chat_template,
|
| 201 |
+
patch_size=patch_size or saved.get("patch_size", [16, 16]),
|
| 202 |
+
max_num_patches=max_num_patches or saved.get("max_num_patches", 1024),
|
| 203 |
+
image_mean=image_mean or saved.get("image_mean", _DEFAULT_MEAN),
|
| 204 |
+
image_std=image_std or saved.get("image_std", _DEFAULT_STD),
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
# ── Compatibility shim ────────────────────────────────────────────────────
|
| 208 |
+
|
| 209 |
+
@property
|
| 210 |
+
def image_processor(self):
|
| 211 |
+
"""
|
| 212 |
+
Shim for code that expects a Qwen2VL-style image_processor (e.g. TokenBudgetDataset).
|
| 213 |
+
NaFlex encodes images via cross-attention ViT — they do NOT add tokens to the text
|
| 214 |
+
sequence, so max_pixels / min_pixels are set to signal zero image-token overhead.
|
| 215 |
+
"""
|
| 216 |
+
patch_pixels = self.patch_h * self.patch_w
|
| 217 |
+
max_pixels = self.max_num_patches * patch_pixels
|
| 218 |
+
|
| 219 |
+
class _Shim:
|
| 220 |
+
pass
|
| 221 |
+
|
| 222 |
+
shim = _Shim()
|
| 223 |
+
shim.max_pixels = max_pixels
|
| 224 |
+
shim.min_pixels = patch_pixels
|
| 225 |
+
# Signal to any caller that images don't contribute to the text token count
|
| 226 |
+
shim.image_tokens_in_text = False
|
| 227 |
+
return shim
|
| 228 |
+
|
| 229 |
+
# ── Image helpers ─────────────────────────────────────────────────────────
|
| 230 |
+
|
| 231 |
+
def _resize(self, img: Image.Image, max_num_patches: Optional[int] = None) -> Image.Image:
|
| 232 |
+
"""
|
| 233 |
+
Resize image so that:
|
| 234 |
+
- H is a multiple of patch_h, W is a multiple of patch_w
|
| 235 |
+
- patch count <= max_num_patches (or self.max_num_patches if not given)
|
| 236 |
+
- aspect ratio is preserved as closely as possible
|
| 237 |
+
"""
|
| 238 |
+
max_n = max_num_patches if max_num_patches is not None else self.max_num_patches
|
| 239 |
+
W_orig, H_orig = img.size
|
| 240 |
+
# Target: scale uniformly so total patches == max_n
|
| 241 |
+
# patches = ceil(H/P_h) * ceil(W/P_w) ≈ (H*W) / (P_h*P_w)
|
| 242 |
+
area_per_patch = self.patch_h * self.patch_w
|
| 243 |
+
scale = math.sqrt(max_n * area_per_patch / (H_orig * W_orig))
|
| 244 |
+
# Round to nearest patch grid boundary
|
| 245 |
+
H_new = max(self.patch_h, round(H_orig * scale / self.patch_h) * self.patch_h)
|
| 246 |
+
W_new = max(self.patch_w, round(W_orig * scale / self.patch_w) * self.patch_w)
|
| 247 |
+
# If already within budget, only snap to grid without upscaling
|
| 248 |
+
if H_orig <= H_new and W_orig <= W_new:
|
| 249 |
+
H_new = max(self.patch_h, round(H_orig / self.patch_h) * self.patch_h)
|
| 250 |
+
W_new = max(self.patch_w, round(W_orig / self.patch_w) * self.patch_w)
|
| 251 |
+
return img.resize((W_new, H_new), Image.BICUBIC)
|
| 252 |
+
|
| 253 |
+
def _patchify(
|
| 254 |
+
self, img: Image.Image, max_num_patches: Optional[int] = None
|
| 255 |
+
) -> tuple[np.ndarray, np.ndarray]:
|
| 256 |
+
"""
|
| 257 |
+
Returns:
|
| 258 |
+
patches: [N, 3 * patch_h * patch_w] float32
|
| 259 |
+
positions: [N, 2] int32 (row, col)
|
| 260 |
+
"""
|
| 261 |
+
img = img.convert("RGB")
|
| 262 |
+
img = self._resize(img, max_num_patches=max_num_patches)
|
| 263 |
+
arr = np.array(img, dtype=np.float32) / 255.0 # [H, W, 3]
|
| 264 |
+
arr = (arr - self.image_mean) / self.image_std # normalise
|
| 265 |
+
|
| 266 |
+
H, W, _ = arr.shape
|
| 267 |
+
n_rows = H // self.patch_h
|
| 268 |
+
n_cols = W // self.patch_w
|
| 269 |
+
# Reshape to [n_rows, n_cols, patch_h, patch_w, 3] then flatten per patch
|
| 270 |
+
arr = arr.reshape(n_rows, self.patch_h, n_cols, self.patch_w, 3)
|
| 271 |
+
arr = arr.transpose(0, 2, 1, 3, 4) # [n_rows, n_cols, P_h, P_w, 3]
|
| 272 |
+
patches = arr.reshape(n_rows * n_cols, self.patch_h * self.patch_w * 3) # channels-last order
|
| 273 |
+
|
| 274 |
+
rows, cols = np.meshgrid(np.arange(n_rows), np.arange(n_cols), indexing="ij")
|
| 275 |
+
positions = np.stack([rows.ravel(), cols.ravel()], axis=-1).astype(np.int32)
|
| 276 |
+
|
| 277 |
+
return patches, positions
|
| 278 |
+
|
| 279 |
+
# ── Main __call__ ─────────────────────────────────────────────────────────
|
| 280 |
+
|
| 281 |
+
def __call__(
|
| 282 |
+
self,
|
| 283 |
+
text: Union[str, List[str], None] = None,
|
| 284 |
+
images: Union[Image.Image, List[Image.Image], None] = None,
|
| 285 |
+
return_tensors: Optional[str] = "pt",
|
| 286 |
+
padding: Union[bool, str, PaddingStrategy] = False,
|
| 287 |
+
truncation: Union[bool, str, TruncationStrategy] = False,
|
| 288 |
+
max_length: Optional[int] = None,
|
| 289 |
+
max_num_patches: Optional[int] = None,
|
| 290 |
+
**kwargs,
|
| 291 |
+
) -> BatchFeature:
|
| 292 |
+
if text is None and images is None:
|
| 293 |
+
raise ValueError("At least one of `text` or `images` must be provided.")
|
| 294 |
+
|
| 295 |
+
# ── Text ──────────────────────────────────────────────────────────────
|
| 296 |
+
encoding = {}
|
| 297 |
+
if text is not None:
|
| 298 |
+
text_enc = self.tokenizer(
|
| 299 |
+
text,
|
| 300 |
+
padding=padding,
|
| 301 |
+
truncation=truncation,
|
| 302 |
+
max_length=max_length,
|
| 303 |
+
return_tensors=return_tensors,
|
| 304 |
+
**kwargs,
|
| 305 |
+
)
|
| 306 |
+
encoding.update(text_enc)
|
| 307 |
+
|
| 308 |
+
# ── Images ────────────────────────────────────────────────────────────
|
| 309 |
+
if images is not None:
|
| 310 |
+
if isinstance(images, Image.Image):
|
| 311 |
+
images = [images]
|
| 312 |
+
|
| 313 |
+
all_patches, all_positions = [], []
|
| 314 |
+
for img in images:
|
| 315 |
+
patches, positions = self._patchify(img, max_num_patches=max_num_patches)
|
| 316 |
+
all_patches.append(patches)
|
| 317 |
+
all_positions.append(positions)
|
| 318 |
+
|
| 319 |
+
# Pad to max N in batch
|
| 320 |
+
max_n = max(p.shape[0] for p in all_patches)
|
| 321 |
+
patch_dim = all_patches[0].shape[1]
|
| 322 |
+
|
| 323 |
+
padded_patches = np.zeros((len(images), max_n, patch_dim), dtype=np.float32)
|
| 324 |
+
padded_positions = np.zeros((len(images), max_n, 2), dtype=np.int32)
|
| 325 |
+
patch_attn_mask = np.zeros((len(images), max_n), dtype=np.bool_)
|
| 326 |
+
|
| 327 |
+
for i, (p, pos) in enumerate(zip(all_patches, all_positions)):
|
| 328 |
+
n = p.shape[0]
|
| 329 |
+
padded_patches[i, :n] = p
|
| 330 |
+
padded_positions[i, :n] = pos
|
| 331 |
+
patch_attn_mask[i, :n] = True
|
| 332 |
+
|
| 333 |
+
if return_tensors == "pt":
|
| 334 |
+
encoding["pixel_values"] = torch.from_numpy(padded_patches)
|
| 335 |
+
encoding["patch_positions"] = torch.from_numpy(padded_positions).long()
|
| 336 |
+
encoding["patch_attention_mask"] = torch.from_numpy(patch_attn_mask)
|
| 337 |
+
else:
|
| 338 |
+
encoding["pixel_values"] = padded_patches
|
| 339 |
+
encoding["patch_positions"] = padded_positions
|
| 340 |
+
encoding["patch_attention_mask"] = patch_attn_mask
|
| 341 |
+
|
| 342 |
+
return BatchFeature(data=encoding, tensor_type=None)
|
optimizer.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:59fc3843edd0919848722667405f25d3cfbe4892864d52f88fd26c842773d59f
|
| 3 |
+
size 39432523
|
processor_config.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"auto_map": {
|
| 3 |
+
"AutoProcessor": "naflex_processor.NaFlexProcessor"
|
| 4 |
+
},
|
| 5 |
+
"image_mean": [
|
| 6 |
+
0.7930999994277954,
|
| 7 |
+
0.7930999994277954,
|
| 8 |
+
0.7930999994277954
|
| 9 |
+
],
|
| 10 |
+
"image_std": [
|
| 11 |
+
0.1738000065088272,
|
| 12 |
+
0.1738000065088272,
|
| 13 |
+
0.1738000065088272
|
| 14 |
+
],
|
| 15 |
+
"max_num_patches": 1024,
|
| 16 |
+
"patch_size": [
|
| 17 |
+
14,
|
| 18 |
+
7
|
| 19 |
+
],
|
| 20 |
+
"processor_class": "NaFlexProcessor"
|
| 21 |
+
}
|
rng_state.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1a0e5159c27f95ddf941e7fdd8971de9b1bf884948c2057943cee9afa8da436a
|
| 3 |
+
size 14645
|
scheduler.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9df532fb062f922e323a8e2a1e905258b664b5e86dd48f4d76284c7b79df6901
|
| 3 |
+
size 1785
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": null,
|
| 3 |
+
"auto_map": {
|
| 4 |
+
"AutoProcessor": "naflex_processor.NaFlexProcessor"
|
| 5 |
+
},
|
| 6 |
+
"backend": "tokenizers",
|
| 7 |
+
"bos_token": "<|im_start|>",
|
| 8 |
+
"clean_up_tokenization_spaces": false,
|
| 9 |
+
"eos_token": "<|im_end|>",
|
| 10 |
+
"errors": "replace",
|
| 11 |
+
"is_local": true,
|
| 12 |
+
"max_length": 512,
|
| 13 |
+
"model_max_length": 131072,
|
| 14 |
+
"pad_to_multiple_of": null,
|
| 15 |
+
"pad_token": "<|endoftext|>",
|
| 16 |
+
"pad_token_type_id": 0,
|
| 17 |
+
"padding_side": "left",
|
| 18 |
+
"processor_class": "NaFlexProcessor",
|
| 19 |
+
"split_special_tokens": false,
|
| 20 |
+
"stride": 0,
|
| 21 |
+
"tokenizer_class": "Qwen2Tokenizer",
|
| 22 |
+
"truncation_side": "right",
|
| 23 |
+
"truncation_strategy": "longest_first",
|
| 24 |
+
"unk_token": null,
|
| 25 |
+
"chat_template": "{%- for message in messages %}\n{{- '<|im_start|>' }\n{%- if message['content'] is string %}\n{{- message['content'] }}\n{%- else %}\n{%- for content in message['content'] %}\n{%- if content['type'] == 'text' %}\n{{- content['text'] }}\n{%- endif %}\n{%- endfor %}\n{%- endif %}\n{{- '<|im_end|>\\n' }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n{{- '<|im_start|>' }}\n{%- endif %}\n"
|
| 26 |
+
}
|
trainer_state.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
training_args.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:898695e4b32a5cd2cb452883a79955464695ac9169096c380fcf0ce1a88b737b
|
| 3 |
+
size 5777
|