d3-nano v3.1.0

MODEL_MANIFEST.json

@@ -554,14 +554,14 @@
  },
  "runtime": {
   "files_sha256": {
-   "d3_runtime.py": "139ece85007c5771b5e374da9658dc8832eb23aabfeaabe29f18e56d59d7f82f",
+   "d3_runtime.py": "6c049aa7d3da8da3738602a2812572879d0a47b03c28c3e33f36d7300257de89",
    "d3_fast.py": "68a1fb9a2c3bde9ac2763bf504decae592ca12722698fbde1b39a31262955535",
    "d3_kernels.py": "8f7534ca1e1846ebf7c4c73058002f3be4dbf0d5d426f0815150ae0cbe7677bd",
-   "modeling_d3.py": "c87cb2c608ce078c8cb2ea8070624ab5a1c4eca02790492bd6e33c2fdc279acb",
-   "pipeline_d3.py": "c0975a91316bcda9c09c5f14bf634ceec94e6980ed7e76a29b718fb451fe4091",
-   "d3_server.py": "11211faa83fee50b957903f0d596c153b0921cb5414f18edee31d8dac76db9fc",
-   "d3_engine.py": "77628954a52cdd03048125899504f1ef55770a768ff8d2c34971a513bfb45a62",
-   "requirements.txt": "3b0f92cb48584015ec0c9adf1559445806bd2ba9f6a41a92be6b2dd07fa48a21",
+   "modeling_d3.py": "f8f8db23ff58f1a990ffa24977d744a50e1b4446d02a64c22070967705b6dbef",
+   "pipeline_d3.py": "4f9297c7c74e596e9b67df91a1e6823cfada48bf781336f141974ff6a2f1038e",
+   "d3_server.py": "f9b8f2481461a074f469b4f58da3b0d7004824096528366d5b1f23187dd6e692",
+   "d3_engine.py": "086238942762df45047a32cd9a8e2bedd9b63167eb40be260c776de567979f53",
+   "requirements.txt": "57a1938448bc240bdf4fee5d1b9cbc8450109fa1f6cee11e94dc04f274be8a32",
    "d3_format.py": "187a0f359f6f89a844e723377769556f78133b118851e702e4b0fe5123ccb0f1"
   }
  },
@@ -578,31 +578,32 @@
    }
   ]
  },
- "built_utc": "2026-10-11T04:21:49+00:00",
+ "built_utc": "2026-10-11T04:33:06+00:00",
  "files_sha256": {
   "LICENSE": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
   "NOTICE": "300b6105c41a1843653e40be644c3d54e06288b25199373e04143530cc3db366",
-  "README.md": "cb20a6644d2e779be1fbb5c22a8e9b5ac2d7c6954554d03348f1753613d1b4ac",
+  "README.md": "65356c023e4a7f6881ba36c761e997f834543caded50d9fc45737176ba49d510",
   "assets/banner.png": "cfd3c0d984881e917b33d22d011a086f5ba7a7d16e95769e5bc03c5e508d790a",
   "assets/example-receipt.png": "b9dbb8b103bacd45f1de824d844a3c07b81d57cb1a7466712ef060ad7dcb3926",
+  "assets/example-video.mp4": "f4968ef89d07e97f20df4fd8047510a7a8e9df5c2d07c35e363a97927f164ab9",
   "assets/index-areas.png": "a290ace23e12c01ae96d0959aa8e715cf01d22e9ff09e3314db74b0f1a4f8449",
   "assets/index-pareto.png": "8df8991d5bd83d82ce20db40318c2fc190e4768c9163e01ce31440194fcde910",
   "chat_template.jinja": "273d8e0e683b885071fb17e08d71e5f2a5ddfb5309756181681de4f5a1822d80",
   "config.json": "9caa6ab8956c859c3852328dda6322893a18b30826d3916db91d938d6c74ac4f",
-  "d3_engine.py": "77628954a52cdd03048125899504f1ef55770a768ff8d2c34971a513bfb45a62",
+  "d3_engine.py": "086238942762df45047a32cd9a8e2bedd9b63167eb40be260c776de567979f53",
   "d3_fast.py": "68a1fb9a2c3bde9ac2763bf504decae592ca12722698fbde1b39a31262955535",
   "d3_format.py": "187a0f359f6f89a844e723377769556f78133b118851e702e4b0fe5123ccb0f1",
   "d3_kernels.py": "8f7534ca1e1846ebf7c4c73058002f3be4dbf0d5d426f0815150ae0cbe7677bd",
-  "d3_runtime.py": "139ece85007c5771b5e374da9658dc8832eb23aabfeaabe29f18e56d59d7f82f",
-  "d3_server.py": "11211faa83fee50b957903f0d596c153b0921cb5414f18edee31d8dac76db9fc",
+  "d3_runtime.py": "6c049aa7d3da8da3738602a2812572879d0a47b03c28c3e33f36d7300257de89",
+  "d3_server.py": "f9b8f2481461a074f469b4f58da3b0d7004824096528366d5b1f23187dd6e692",
   "decision_config.json": "9175a0f42d5886def9c8429fdf8b0be58a3c97d106b94520e2ac38fe326302bf",
   "merges.txt": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d",
   "model.safetensors": "7269b3260f1106f3443bfdb028a76143be2e679db3f29a31f4dfd1d595f03155",
-  "modeling_d3.py": "c87cb2c608ce078c8cb2ea8070624ab5a1c4eca02790492bd6e33c2fdc279acb",
-  "pipeline_d3.py": "c0975a91316bcda9c09c5f14bf634ceec94e6980ed7e76a29b718fb451fe4091",
+  "modeling_d3.py": "f8f8db23ff58f1a990ffa24977d744a50e1b4446d02a64c22070967705b6dbef",
+  "pipeline_d3.py": "4f9297c7c74e596e9b67df91a1e6823cfada48bf781336f141974ff6a2f1038e",
   "preprocessor_config.json": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516",
   "readout.safetensors": "662e8a26d8016b2d479f1ca66bf64944603316c2b5e074ab7ac3304b8ae045e9",
-  "requirements.txt": "3b0f92cb48584015ec0c9adf1559445806bd2ba9f6a41a92be6b2dd07fa48a21",
+  "requirements.txt": "57a1938448bc240bdf4fee5d1b9cbc8450109fa1f6cee11e94dc04f274be8a32",
   "tokenizer.json": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42",
   "tokenizer_config.json": "49e2b6e395f959f077f1e992b338919c0d4a9732fc6e613995e06557f843500c",
   "video_preprocessor_config.json": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13",
@@ -611,27 +612,28 @@
  "files_bytes": {
   "LICENSE": 11357,
   "NOTICE": 104,
-  "README.md": 6258,
+  "README.md": 7846,
   "assets/banner.png": 2233242,
   "assets/example-receipt.png": 65918,
+  "assets/example-video.mp4": 26300,
   "assets/index-areas.png": 124987,
   "assets/index-pareto.png": 184438,
   "chat_template.jinja": 7755,
   "config.json": 2885,
-  "d3_engine.py": 4227,
+  "d3_engine.py": 4413,
   "d3_fast.py": 22331,
   "d3_format.py": 4881,
   "d3_kernels.py": 16129,
-  "d3_runtime.py": 59432,
-  "d3_server.py": 8333,
+  "d3_runtime.py": 88800,
+  "d3_server.py": 10100,
   "decision_config.json": 5502,
   "merges.txt": 3353259,
   "model.safetensors": 4426555160,
-  "modeling_d3.py": 11673,
-  "pipeline_d3.py": 2601,
+  "modeling_d3.py": 12079,
+  "pipeline_d3.py": 2936,
   "preprocessor_config.json": 390,
   "readout.safetensors": 2089040,
-  "requirements.txt": 707,
+  "requirements.txt": 679,
   "tokenizer.json": 12807982,
   "tokenizer_config.json": 16709,
   "video_preprocessor_config.json": 385,

README.md

@@ -10,6 +10,7 @@ tags:
 - system-one
 - multimodal
 - vision
+- video
 - safetensors
 ---
 
@@ -17,12 +18,12 @@ tags:
 
 # d3-nano
 
-**d3-nano** is the 2B multimodal foundation decision model of Decision 3.0, the decision models of [vLLM Semantic Router](https://github.com/vllm-project/semantic-router). Give it an input (text or JSON, optionally with images) and the questions you need answered: pick one of several options, say yes or no, or rate on a scale. It answers them all in one call and returns a probability for every answer, without generating text.
+**d3-nano** is the 2B multimodal foundation decision model of Decision 3.0, the decision models of [vLLM Semantic Router](https://github.com/vllm-project/semantic-router). Give it an input (text or JSON, optionally with images and videos) and the questions you need answered: pick one of several options, say yes or no, or rate on a scale. It answers them all in one call and returns a probability for every answer, without generating text.
 
 | | |
 | --- | --- |
 | **Parameters** | 2.21B, including the 0.33B vision encoder |
-| **Inputs** | Text or JSON, plus images (several per request) |
+| **Inputs** | Text or JSON, plus images and videos (several per request) |
 | **Decision types** | Choice · Yes / No · Score |
 | **License** | Apache-2.0 |
 
@@ -31,13 +32,14 @@ tags:
 - **Jev Decision Index 0.3, public suite: 42.22**, measured with the official 0.3 kit on the released weights: all 140,178 public requests answered, none unsupported.
 - **+12.2 on the public suite over Decision 2.0** (its 2B model: 30.05 on the board), ahead in all five areas.
 - **Reads images:** multiple images per request (PNG, JPEG or WebP), given as paths, URLs, PIL images or base64 data URLs; every question of the request sees all of them.
-- **Speed:** a median of 12.9 ms for a text request and 73.8 ms for a request with an image, on one AMD Instinct MI325X GPU, one request at a time.
+- **Reads videos:** multiple videos per request (MP4, WebM, MOV or MKV), given as paths, URLs, base64 data URLs or frame arrays, read at 2 frames per second; **Perception Test (multiple-choice video QA): 63.5** (internal evaluation).
+- **Speed:** a median of 12.9 ms for a text request, 73.8 ms for a request with an image and 344.9 ms for a request with a 10-second video, on one AMD Instinct MI325X GPU, one request at a time.
 - **Many questions, one call:** Choice, Yes / No and Score questions about the same input are answered together, each from its own forward pass over the input, with a probability for every option.
 
 ## Quickstart
 
 ```bash
-pip install "transformers==5.17.0" torch torchvision pillow safetensors accelerate
+pip install "transformers==5.17.0" torch torchvision pillow opencv-python-headless safetensors accelerate
 pip install flash-linear-attention  # optional: fast GPU kernels for the linear-attention layers
 ```
 
@@ -111,11 +113,34 @@ result = model.system_one(
 )
 print(json.dumps(result["answers"], indent=2))
 
+# Text and a video (or several)
+clip = hf_hub_download("vllm-sr/d3-nano", "assets/example-video.mp4")
+result = model.system_one(
+    state="A short clip from a test camera.",
+    videos=[clip],  # local paths, http(s) URLs, base64 data URLs or frame arrays
+    questions={
+        "direction": {
+            "type": "choice",
+            "instructions": "Which way does the square move?",
+            "criteria": {
+                "right": "From left to right",
+                "left": "From right to left",
+                "still": "It does not move"
+            }
+        },
+        "color_change": {
+            "type": "noul",
+            "instructions": "Does the square change color?"
+        }
+    },
+)
+print(json.dumps(result["answers"], indent=2))
+
 # Or as a pipeline:
-# transformers.pipeline("decision", model="vllm-sr/d3-nano", trust_remote_code=True)(state=..., questions=..., images=...)
+# transformers.pipeline("decision", model="vllm-sr/d3-nano", trust_remote_code=True)(state=..., questions=..., images=..., videos=...)
 ```
 
-Images go before the text of the request, each read at up to 1.6 megapixels; every question of the request sees all of them.
+Images go before the text of the request, each read at up to 1.6 megapixels; videos follow the images, read at 2 frames per second (at most 32 frames spread over the whole video, each at up to 0.2 megapixels). Every question of the request sees all of them.
 
 ## Evaluation
 
@@ -157,6 +182,18 @@ d3-nano on the public vision benchmarks († approximate rebuild):
 | R-Bench-M | 14.8 |
 | MMMU-Pro vision | 18.6 |
 
+### Videos: Perception Test
+
+| Perception Test (validation) | d3-nano |
+| --- | ---: |
+| **All 19,140 questions** | **63.5** |
+| Memory | 53.8 |
+| Abstraction | 55.7 |
+| Physics | 56.0 |
+| Semantics | 81.0 |
+
+Multiple-choice video question answering (three options per question, chance 33.3): top-1 accuracy (%), videos read with the defaults above.
+
 <sub>d3-nano: internal evaluation. Others: live board data, text 2026-10-10, vision 2026-10-09.</sub>
 
 ## License

d3_engine.py

@@ -11,7 +11,8 @@ average, see d3_runtime.py). A request with a question over the checkpoint's inp
 (nothing is truncated).
 
 Image requests: ``engine(state, questions, images=[...])`` with any number of images (PIL images, paths,
-http(s) or data URLs) that every question sees.
+http(s) or data URLs) that every question sees. Video requests: ``engine(state, questions, videos=[...])``
+(with or without images) with any number of videos (paths, http(s) or data URLs, frame arrays).
 """
 
 from __future__ import annotations
@@ -37,8 +38,8 @@ class D3Engine(Engine):
     name = "d3"
     latency = (
         "Device-synchronized in-process request wall time including prompt rendering and tokenization (and, "
-        "for image requests, image decoding and preprocessing); one request per call, its questions in request "
-        "order in batches of batch_size; excludes model loading."
+        "for image and video requests, their decoding and preprocessing); one request per call, its questions in "
+        "request order in batches of batch_size; excludes model loading."
     )
 
     def __init__(
@@ -90,8 +91,8 @@ class D3Engine(Engine):
     def synchronize(self):
         self.decision.synchronize()
 
-    def __call__(self, state, questions, images=None):
-        prepared = self.decision.prepare(state, questions, images)
+    def __call__(self, state, questions, images=None, videos=None):
+        prepared = self.decision.prepare(state, questions, images, videos)
         over = [
             e["message"]
             for e in prepared.errors.values()

d3_runtime.py

@@ -11,6 +11,7 @@ probability for every option. No text is generated and no input is truncated.
     #                                     "confidence": ...}}, "usage": {"input_tokens": n, "output_tokens": 0}}
 
     model.system_one(state="...", questions={...}, images=["photo.png", "label.jpg"])
+    model.system_one(state="...", questions={...}, videos=["clip.mp4"])
 
 The checkpoint directory holds ``config.json`` + ``model*.safetensors`` (a transformers
 ``Qwen3_5Model``, or a ``Qwen3VLModel`` when ``config.json`` says ``model_type: qwen3_vl``),
@@ -26,6 +27,14 @@ the aspect ratio; each 32 x 32 pixel patch is one input token. The prompt is the
 images in front; a request without images takes exactly the text path. Image inputs need a checkpoint
 with the vision tower (``visual.*`` weights).
 
+Videos: a request also takes any number of videos (local paths, http(s) URLs, base64 ``data:video/...``
+URLs, or frame arrays: a uint8 ``[frames, height, width, 3]`` RGB array or a list of PIL images, read as
+frames sampled at 2 per second), shared by all of its questions and placed after the images. Files are
+decoded with OpenCV (FFmpeg): 2 frames per second, at least 4 and at most 32 frames spread evenly over the
+whole video, each read by the checkpoint's own video processor at up to 200,704 pixels (0.2 MP); every two
+frames are one group of input tokens (one token per 32 x 32 pixels) after a timestamp. All videos of one
+request take at most 16,384 input tokens. A request without videos takes exactly the text or image path.
+
 Numerics: BF16 backbone with SDPA attention, FP32 readout and softmax (unless the checkpoint says
 otherwise). A request's questions run in request order, ``batch_size`` per forward pass, each batch
 left-padded to its longest prompt. ``noncausal_full_attention`` (Qwen3.5 backbones only) lets the
@@ -122,6 +131,22 @@ IMAGE_INPUTS = (
     "pixel_values",
     "image_grid_thw",
 )
+VIDEO_FPS = 2.0
+VIDEO_MIN_FRAMES = 4
+VIDEO_MAX_FRAMES = 32
+# Per frame; one input token per 32 x 32 pixels for every two frames.
+VIDEO_MAX_PIXELS = 200_704
+VIDEO_MIN_PIXELS = 4_096
+# All videos of one request.
+VIDEO_MAX_TOKENS = 16_384
+# Checked for every video the server receives (strict loading).
+MAX_VIDEO_BYTES = 32_000_000
+MAX_VIDEO_SOURCE_PIXELS = 8_294_400
+MAX_VIDEO_SECONDS = 300
+VIDEO_FORMATS = ("mp4", "webm", "quicktime", "x-matroska")
+# A source frame rate outside this range is treated as unknown.
+VIDEO_SOURCE_FPS = (0.1, 1000.0)
+VIDEO_UNKNOWN_FPS = 24.0
 
 
 class MaxLengthExceeded(ValueError):
@@ -239,6 +264,28 @@ def image_messages(
     raise ValueError(f"unknown prompt family {prompt!r}")
 
 
+def video_messages(
+    prompt: str,
+    state: Any,
+    question: dict[str, Any],
+    codes: Sequence[str],
+    n_images: int,
+    n_videos: int,
+) -> list[dict[str, Any]]:
+    """The prompt family's messages with the image placeholders, then the video placeholders, before the user text."""
+    if n_videos < 1:
+        raise ValueError(f"a video prompt takes at least 1 video, got {n_videos}")
+    media: list[dict[str, Any]] = [{"type": "image"} for _ in range(n_images)]
+    media += [{"type": "video"} for _ in range(n_videos)]
+    if prompt == "d3":
+        text = {"type": "text", "text": user_prompt(state, question, codes)}
+        return [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content": media + [text]},
+        ]
+    raise ValueError(f"unknown prompt family {prompt!r}")
+
+
 def reversed_question(question: Question) -> dict[str, Any] | None:
     """The rendered choice question with its options in reverse order (None when there is nothing to permute)."""
     if question.kind != "choice" or len(question.keys) < 2:
@@ -332,7 +379,7 @@ def _data_url_payload(value: str, strict: bool) -> bytes:
         raise ValueError("invalid base64 image data") from exc
 
 
-def _download(url: str) -> bytes:
+def _download(url: str, what: str = "image") -> bytes:
     import urllib.request
 
     request = urllib.request.Request(url, headers={"User-Agent": RUNTIME})
@@ -340,7 +387,7 @@ def _download(url: str) -> bytes:
         payload = response.read(MAX_DOWNLOAD_BYTES + 1)
     if len(payload) > MAX_DOWNLOAD_BYTES:
         raise ValueError(
-            f"the image at {url} is larger than {MAX_DOWNLOAD_BYTES:,} bytes"
+            f"the {what} at {url} is larger than {MAX_DOWNLOAD_BYTES:,} bytes"
         )
     return payload
 
@@ -450,6 +497,278 @@ def linearize_patch_embed(model) -> int:
     return count
 
 
+# ---------------------------------------------------------------------------------------------
+# Videos
+# ---------------------------------------------------------------------------------------------
+
+
+@dataclass
+class Video:
+    """A decoded video: the sampled RGB frames and where they come from in the source.
+
+    ``frames`` is a uint8 array ``[frames, height, width, 3]`` holding an even number of frames (the last
+    one is repeated when needed); ``indices`` are their source frame numbers and ``fps`` the source frame
+    rate, which give each frame its timestamp.
+    """
+
+    frames: Any
+    fps: float
+    indices: list[int]
+    total_frames: int
+
+    @property
+    def size(self) -> tuple[int, int]:
+        """(width, height) of the frames."""
+        return int(self.frames.shape[2]), int(self.frames.shape[1])
+
+    def metadata(self) -> dict[str, Any]:
+        """The video processor's ``video_metadata`` for these frames."""
+        width, height = self.size
+        return {
+            "total_num_frames": self.total_frames,
+            "fps": self.fps,
+            "width": width,
+            "height": height,
+            "duration": self.total_frames / self.fps,
+            "frames_indices": list(self.indices),
+        }
+
+
+def video_sample_indices(total: int, fps: float) -> list[int]:
+    """Source frames to read: 2 per second, at least 4 and at most 32, spread evenly over the whole video."""
+    import numpy as np
+
+    count = int(total / fps * VIDEO_FPS)
+    count = min(max(count, VIDEO_MIN_FRAMES), VIDEO_MAX_FRAMES, total)
+    return np.linspace(0, total - 1, count).round().astype(int).tolist()
+
+
+def _video(frames: list[Any], indices: list[int], fps: float, total: int) -> Video:
+    import numpy as np
+
+    if len(frames) % 2:
+        frames.append(frames[-1])
+        indices.append(indices[-1])
+    try:
+        stacked = np.stack(frames)
+    except ValueError as exc:
+        raise ValueError("all frames of a video must have the same size") from exc
+    return Video(np.ascontiguousarray(stacked), float(fps), list(indices), int(total))
+
+
+def _video_data_url_payload(value: str, strict: bool) -> bytes:
+    header, separator, encoded = value.partition(",")
+    kind = header.strip().lower()
+    if (
+        not separator
+        or not kind.startswith("data:video/")
+        or not kind.endswith(";base64")
+    ):
+        raise ValueError("a data URL video is data:video/<format>;base64,<data>")
+    if strict:
+        if kind[len("data:video/") :].split(";")[0] not in VIDEO_FORMATS:
+            raise ValueError(
+                "videos must be base64 MP4, WebM, QuickTime or Matroska data URLs"
+            )
+        if len(encoded) > 4 * -(-MAX_VIDEO_BYTES // 3):
+            raise ValueError(f"each video must be at most {MAX_VIDEO_BYTES:,} bytes")
+    try:
+        payload = base64.b64decode(encoded, validate=True)
+    except binascii.Error as exc:
+        raise ValueError("invalid base64 video data") from exc
+    if strict and len(payload) > MAX_VIDEO_BYTES:
+        raise ValueError(f"each video must be at most {MAX_VIDEO_BYTES:,} bytes")
+    return payload
+
+
+def _decode_video(
+    source: bytes | str, *, strict: bool, end_frame: int | None = None
+) -> Video:
+    """Decode a video file (path) or encoded video (bytes) with OpenCV and sample its frames."""
+    try:
+        import cv2
+    except ImportError as exc:
+        raise ValueError(
+            "decoding video files needs OpenCV (pip install opencv-python-headless)"
+        ) from exc
+    import tempfile
+
+    try:
+        api = (
+            cv2.CAP_FFMPEG
+            if cv2.videoio_registry.hasBackend(cv2.CAP_FFMPEG)
+            else cv2.CAP_ANY
+        )
+    except (AttributeError, cv2.error):
+        api = cv2.CAP_ANY
+    temporary = None
+    if not isinstance(source, str):
+        # Encoded bytes are read from a temporary file, so they decode exactly as the same file given by path.
+        with tempfile.NamedTemporaryFile(prefix="d3-video-", delete=False) as handle:
+            handle.write(source)
+        source = temporary = handle.name
+
+    def capture():
+        return cv2.VideoCapture(source, api)
+
+    frames, kept, reader = [], [], None
+    try:
+        reader = capture()
+        if not reader.isOpened():
+            raise ValueError("the video could not be opened")
+        width = int(reader.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(reader.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        if width <= 0 or height <= 0:
+            raise ValueError("the video has no frame size")
+        if strict and width * height > MAX_VIDEO_SOURCE_PIXELS:
+            raise ValueError(
+                f"video frames must have at most {MAX_VIDEO_SOURCE_PIXELS:,} pixels"
+            )
+        fps = float(reader.get(cv2.CAP_PROP_FPS) or 0.0)
+        if not (
+            math.isfinite(fps) and VIDEO_SOURCE_FPS[0] <= fps <= VIDEO_SOURCE_FPS[1]
+        ):
+            fps = VIDEO_UNKNOWN_FPS
+        total = int(reader.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
+        if total <= 0:  # the container does not say: count the frames, then read again
+            limit = int(MAX_VIDEO_SECONDS * fps) if strict else None
+            while reader.grab():
+                total += 1
+                if limit is not None and total > limit:
+                    break
+            reader.release()
+            reader = capture()
+        if end_frame is not None:
+            total = min(total, int(end_frame))
+        if total <= 0:
+            raise ValueError("the video has no frames")
+        if strict and total / fps > MAX_VIDEO_SECONDS:
+            raise ValueError(
+                f"each video must be at most {MAX_VIDEO_SECONDS} seconds long"
+            )
+        indices = video_sample_indices(total, fps)
+        wanted = set(indices)
+        for number in range(indices[-1] + 1):
+            if not reader.grab():
+                continue
+            if number in wanted:
+                ok, frame = reader.retrieve()
+                if ok and frame is not None and frame.size:
+                    frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+                    kept.append(number)
+    except cv2.error as exc:
+        raise ValueError(f"invalid video data ({exc.__class__.__name__})") from exc
+    finally:
+        if reader is not None:
+            reader.release()
+        if temporary is not None:
+            os.unlink(temporary)
+    if not frames:
+        raise ValueError("no frame of the video could be decoded")
+    return _video(frames, kept, fps, total)
+
+
+def _frames_video(value: Any, end_frame: int | None = None) -> Video:
+    """A frame array as a video sampled at ``VIDEO_FPS``, thinned evenly to at most ``VIDEO_MAX_FRAMES``."""
+    import numpy as np
+    from PIL import Image
+
+    if hasattr(value, "detach"):
+        value = value.detach().cpu().numpy()
+    if isinstance(value, (list, tuple)):
+        if not value:
+            raise ValueError("a video given as frames needs at least one frame")
+        value = [
+            (
+                np.asarray(frame.convert("RGB"))
+                if isinstance(frame, Image.Image)
+                else np.asarray(frame)
+            )
+            for frame in value
+        ]
+        if len({frame.shape for frame in value}) != 1:
+            raise ValueError("all frames of a video must have the same size")
+        value = np.stack(value)
+    array = np.asarray(value)
+    if (
+        array.ndim != 4
+        or array.shape[-1] != 3
+        or array.dtype != np.uint8
+        or not len(array)
+    ):
+        raise ValueError(
+            "a video given as frames is a uint8 [frames, height, width, 3] RGB array or a list of "
+            "PIL images or [height, width, 3] arrays"
+        )
+    if end_frame is not None:
+        array = array[: max(1, int(end_frame))]
+    total = len(array)
+    if total > VIDEO_MAX_FRAMES:
+        indices = (
+            np.linspace(0, total - 1, VIDEO_MAX_FRAMES).round().astype(int).tolist()
+        )
+    else:
+        indices = list(range(total))
+    return _video([array[i] for i in indices], indices, VIDEO_FPS, total)
+
+
+def load_video(
+    value: Any, *, strict: bool = False, end_frame: int | None = None
+) -> Video:
+    """One video input, decoded and sampled (see the module docstring).
+
+    ``value`` is a local path, an http(s) URL, a ``data:video/<format>;base64,`` URL, a frame array (a uint8
+    ``[frames, height, width, 3]`` RGB array or a list of PIL images or ``[height, width, 3]`` arrays, read
+    as frames sampled at 2 per second) or a ``Video`` this function returned. ``end_frame`` reads only the
+    frames before it. ``strict`` (the server) accepts only data URLs of MP4, WebM, QuickTime or Matroska
+    videos of at most 32,000,000 bytes, 300 seconds and 8,294,400 pixels per frame.
+    """
+    if isinstance(value, Video) and not strict:
+        return value
+    if isinstance(value, os.PathLike):
+        value = os.fspath(value)
+    if not isinstance(value, str):
+        if strict:
+            raise ValueError(
+                "videos must be base64 MP4, WebM, QuickTime or Matroska data URLs"
+            )
+        return _frames_video(value, end_frame)
+    if not value:
+        raise ValueError(
+            "a video is a path, an http(s) URL, a data:video/...;base64 URL or a frame array"
+        )
+    if strict and not value.startswith("data:"):
+        raise ValueError(
+            "videos must be base64 MP4, WebM, QuickTime or Matroska data URLs"
+        )
+    try:
+        if value.startswith("data:"):
+            source: bytes | str = _video_data_url_payload(value, strict)
+        elif value.startswith(("http://", "https://")):
+            source = _download(value, "video")
+        else:
+            path = Path(value).expanduser()
+            if not path.is_file():
+                raise ValueError(f"no video file at {value}")
+            source = str(path)
+    except OSError as exc:
+        raise ValueError(f"video unreadable ({exc})") from exc
+    return _decode_video(source, strict=strict, end_frame=end_frame)
+
+
+def configure_video_processor(processor) -> None:
+    """Read already-sampled frames at up to ``VIDEO_MAX_PIXELS`` each with the checkpoint's video processor."""
+    video = processor.video_processor
+    factor = video.patch_size * video.merge_size
+    video.size = {
+        "shortest_edge": VIDEO_MIN_PIXELS,
+        "longest_edge": VIDEO_MAX_PIXELS * VIDEO_MAX_FRAMES,
+    }
+    video.cap_pixels_per_frame = True
+    video.max_video_tokens = VIDEO_MAX_PIXELS // factor**2
+    video.do_sample_frames = False
+
+
 # ---------------------------------------------------------------------------------------------
 # Package files
 # ---------------------------------------------------------------------------------------------
@@ -684,7 +1003,9 @@ class Prepared:
 
     A text request holds token ``sequences``; an image request holds the decoded ``images`` (shared by
     every question), the rendered prompt ``texts`` and the planned input ``lengths`` (text tokens plus
-    image tokens), and the processor tokenizes it when it runs.
+    image tokens), and the processor tokenizes it when it runs. A video request also holds the decoded
+    ``videos``, their processed pixels (``media``, one copy per image and video) and each video's
+    placeholder expansion (``video_texts``).
     """
 
     keys: list[str]
@@ -694,6 +1015,9 @@ class Prepared:
     images: list[Any] = field(default_factory=list)
     texts: dict[str, str] = field(default_factory=dict)
     lengths: dict[str, int] = field(default_factory=dict)
+    videos: list[Video] = field(default_factory=list)
+    media: dict[str, Any] = field(default_factory=dict)
+    video_texts: list[str] = field(default_factory=list)
     # Reversed-option prompts of the choice questions (permutation_average only).
     reversed_sequences: dict[str, list[int]] = field(default_factory=dict)
     reversed_texts: dict[str, str] = field(default_factory=dict)
@@ -832,6 +1156,22 @@ class D3:
                 self.image_unavailable = (
                     f"the image processor failed to load ({type(exc).__name__}: {exc})"
                 )
+        self.video_unavailable: str | None = None
+        if self.image_unavailable is not None:
+            self.video_unavailable = self.image_unavailable
+        elif self.backbone_type != "qwen3_5":
+            self.video_unavailable = "video inputs need a Qwen3.5 backbone"
+        elif getattr(self.processor, "video_processor", None) is None:
+            self.video_unavailable = "the checkpoint has no video processor"
+        else:
+            try:
+                configure_video_processor(self.processor)
+            except (
+                Exception
+            ) as exc:  # noqa: BLE001 - text and image requests do not use it
+                self.video_unavailable = (
+                    f"the video processor failed to load ({type(exc).__name__}: {exc})"
+                )
         fast, reason = d3_fast, "d3_fast.py is not present"
         if fast is None:
             try:
@@ -900,6 +1240,36 @@ class D3:
             enable_thinking=False,
         )
 
+    def video_text(
+        self, state: Any, question: dict[str, Any], n_images: int, n_videos: int
+    ) -> str:
+        return self.processor.apply_chat_template(
+            video_messages(
+                self.prompt, state, question, self.codes, n_images, n_videos
+            ),
+            tokenize=False,
+            add_generation_prompt=True,
+            enable_thinking=False,
+        )
+
+    def load_videos(
+        self, videos: Sequence[Any], *, strict: bool = False
+    ) -> list[Video]:
+        """Decode and sample a request's videos, any number; raises ValueError for a video it cannot read."""
+        if not videos:
+            return []
+        if self.video_unavailable is not None:
+            raise ValueError(
+                f"video inputs are not available: {self.video_unavailable}"
+            )
+        decoded = []
+        for number, value in enumerate(videos):
+            try:
+                decoded.append(load_video(value, strict=strict))
+            except ValueError as exc:
+                raise ValueError(f"videos[{number}]: {exc}") from exc
+        return decoded
+
     def load_images(self, images: Sequence[Any], *, strict: bool = False) -> list[Any]:
         """Decode a request's images (RGB PIL), any number; raises ValueError for an image it cannot read."""
         if not images:
@@ -921,11 +1291,14 @@ class D3:
         state: Any,
         questions: Mapping[str, Any],
         images: Sequence[Any] | None = None,
+        videos: Sequence[Any] | None = None,
     ) -> Prepared:
-        """Validate and tokenize one request; malformed ``state``, ``questions`` or ``images`` raise ValueError.
+        """Validate and tokenize one request; malformed ``state``, ``questions``, ``images`` or ``videos`` raise
+        ValueError.
 
-        ``images`` (a list of any number of images shared by every question) select the image path. Without
-        images the request takes the text path unchanged.
+        ``images`` (a list of any number of images shared by every question) select the image path, ``videos``
+        (a list of any number of videos, with or without images) the video path. Without them the request takes
+        the text path unchanged.
         """
         if not isinstance(questions, Mapping) or not questions:
             raise ValueError(
@@ -936,6 +1309,15 @@ class D3:
         _json_value(state)
         if images is not None and not isinstance(images, (list, tuple)):
             raise ValueError("images must be a list of images")
+        if videos is not None and not isinstance(videos, (list, tuple)):
+            raise ValueError("videos must be a list of videos")
+        if videos:
+            return self._prepare_videos(
+                state,
+                questions,
+                self.load_images(images or []),
+                self.load_videos(videos),
+            )
         if images:
             return self._prepare_images(state, questions, self.load_images(images))
         prepared = Prepared(keys=list(questions))
@@ -976,9 +1358,10 @@ class D3:
     ) -> None:
         """Render and tokenize the reversed-option prompt of every runnable choice question.
 
-        ``visual``: the image tokens of the request (image path), counted like ``_prepare_images`` counts them.
+        ``visual``: the image and video tokens of the request (image and video paths), counted like
+        ``_prepare_images`` and ``_prepare_videos`` count them.
         """
-        n_images = len(prepared.images)
+        n_images, n_videos = len(prepared.images), len(prepared.videos)
         texts = []
         for key in prepared.runnable:
             flipped = reversed_question(prepared.questions[key])
@@ -987,18 +1370,22 @@ class D3:
                     (
                         key,
                         (
-                            self.image_text(state, flipped, n_images)
-                            if n_images
-                            else self.text(state, flipped)
+                            self.video_text(state, flipped, n_images, n_videos)
+                            if n_videos
+                            else (
+                                self.image_text(state, flipped, n_images)
+                                if n_images
+                                else self.text(state, flipped)
+                            )
                         ),
                     )
                 )
         if not texts:
             return
-        tokenizer = self.processor.tokenizer if n_images else self.tokenizer
+        tokenizer = self.processor.tokenizer if n_images or n_videos else self.tokenizer
         ids = tokenizer([t for _, t in texts], add_special_tokens=False)["input_ids"]
         for (key, text), sequence in zip(texts, ids):
-            length = len(sequence) - n_images + visual
+            length = len(sequence) - n_images - n_videos + visual
             if self.max_length is not None and length > self.max_length:
                 for planned in (prepared.sequences, prepared.texts, prepared.lengths):
                     planned.pop(key, None)
@@ -1008,7 +1395,7 @@ class D3:
                     "message": f"the reversed-option prompt has {length} tokens, over the maximum context "
                     f"length of {self.max_length} tokens; nothing was truncated",
                 }
-            elif n_images:
+            elif n_images or n_videos:
                 prepared.reversed_texts[key] = text
                 prepared.reversed_lengths[key] = length
             else:
@@ -1069,6 +1456,125 @@ class D3:
             self._prepare_reversed(state, prepared, sum(visual))
         return prepared
 
+    def _prepare_videos(
+        self,
+        state: Any,
+        questions: Mapping[str, Any],
+        images: list[Any],
+        videos: list[Video],
+    ) -> Prepared:
+        """Process the images and videos once, render every question with them in front and plan its tokens."""
+        processor = self.processor
+        try:
+            visual = [
+                visual_tokens(processor.image_processor, *image.size)
+                for image in images
+            ]
+        except ValueError as exc:
+            raise ValueError(f"image rejected by the processor: {exc}") from exc
+        merge = processor.video_processor.merge_size**2
+        try:
+            planned = (
+                sum(
+                    processor.video_processor.get_num_of_video_patches(
+                        len(video.frames), video.size[1], video.size[0]
+                    )
+                    for video in videos
+                )
+                // merge
+            )
+        except ValueError as exc:
+            raise ValueError(f"video rejected by the processor: {exc}") from exc
+        if planned > VIDEO_MAX_TOKENS:
+            raise ValueError(
+                f"the videos take {planned:,} input tokens, over the {VIDEO_MAX_TOKENS:,} tokens a "
+                "request may spend on videos; send fewer or shorter videos"
+            )
+        try:
+            media = dict(
+                processor.video_processor(
+                    videos=[video.frames for video in videos],
+                    video_metadata=[video.metadata() for video in videos],
+                    return_metadata=True,
+                    return_tensors="pt",
+                )
+            )
+        except ValueError as exc:
+            raise ValueError(f"video rejected by the processor: {exc}") from exc
+        video_texts = [
+            processor.replace_video_token(media, video_idx=i)
+            for i in range(len(videos))
+        ]
+        if int(media["video_grid_thw"].prod(-1).sum()) // merge != planned:
+            raise RuntimeError("the video processor and its token count disagree")
+        expanded = sum(
+            len(ids)
+            for ids in processor.tokenizer(video_texts, add_special_tokens=False)[
+                "input_ids"
+            ]
+        )
+        media = {
+            "pixel_values_videos": media["pixel_values_videos"],
+            "video_grid_thw": media["video_grid_thw"],
+        }
+        if images:
+            media.update(processor.image_processor(images=images, return_tensors="pt"))
+        prepared = Prepared(
+            keys=list(questions),
+            images=images,
+            videos=videos,
+            media=media,
+            video_texts=video_texts,
+        )
+        texts = []
+        for key, question in questions.items():
+            try:
+                normalized = normalize_question(question)
+                text = self.video_text(
+                    state, normalized.rendered, len(images), len(videos)
+                )
+            except ValueError as exc:
+                kind = question.get("type") if isinstance(question, Mapping) else None
+                prepared.errors[key] = {
+                    "type": kind,
+                    "error": "invalid_question",
+                    "message": str(exc),
+                }
+                continue
+            if text.count(IMAGE_TOKEN) != len(images) or text.count(VIDEO_TOKEN) != len(
+                videos
+            ):
+                prepared.errors[key] = {
+                    "type": normalized.kind,
+                    "error": "invalid_question",
+                    "message": "the state or question contains a literal image or video placeholder token",
+                }
+                continue
+            prepared.questions[key] = normalized
+            texts.append((key, text))
+        if texts:
+            ids = processor.tokenizer([t for _, t in texts], add_special_tokens=False)[
+                "input_ids"
+            ]
+            for (key, text), sequence in zip(texts, ids):
+                length = (
+                    len(sequence) - len(images) - len(videos) + sum(visual) + expanded
+                )
+                if self.max_length is not None and length > self.max_length:
+                    prepared.errors[key] = {
+                        "type": prepared.questions[key].kind,
+                        "error": "max_length_exceeded",
+                        "message": f"the question prompt has {length} tokens ({sum(visual) + expanded} for "
+                        f"{len(images)} image(s) and {len(videos)} video(s)), over the maximum context length "
+                        f"of {self.max_length} tokens; nothing was truncated",
+                    }
+                    continue
+                prepared.texts[key] = text
+                prepared.lengths[key] = length
+        if self.permutation_average:
+            self._prepare_reversed(state, prepared, sum(visual) + expanded)
+        return prepared
+
     def logits(self, sequences: Sequence[Sequence[int]], counts: Sequence[int]):
         """Masked code logits (FP32, [B, 255]) for one left-padded batch."""
         torch = self.torch
@@ -1196,8 +1702,170 @@ class D3:
             prepared.reversed_lengths.values()
         )
 
+    def media_features(self, media: Mapping[str, Any]) -> dict[str, list[Any]]:
+        """Vision-tower features of each image and video of a video request, computed once per request."""
+        torch = self.torch
+        backbone = self.backbone
+        features: dict[str, list[Any]] = {}
+        with torch.inference_mode(), sdpa_backends(self.device):
+            if "pixel_values" in media:
+                features["image"] = list(
+                    backbone.get_image_features(
+                        media["pixel_values"].to(self.device),
+                        media["image_grid_thw"].to(self.device),
+                        return_dict=True,
+                    ).pooler_output
+                )
+            features["video"] = list(
+                backbone.get_video_features(
+                    media["pixel_values_videos"].to(self.device),
+                    media["video_grid_thw"].to(self.device),
+                    return_dict=True,
+                ).pooler_output
+            )
+        return features
+
+    def video_logits(
+        self,
+        texts: Sequence[str],
+        prepared: Prepared,
+        features: Mapping[str, list[Any]],
+        counts: Sequence[int],
+        width: int,
+    ):
+        """Masked code logits (FP32, [B, 255]) for one left-padded batch of prompts that share the request's videos.
+
+        Each prompt's placeholders are expanded with the request's images and videos and the batch is padded to
+        ``width`` tokens (the longest planned prompt); the vision features of the request go into every prompt,
+        then the text layers run as for an image batch (the fused layers when the fast path is active).
+        """
+        torch = self.torch
+        processor = self.processor
+        backbone = self.backbone
+        media = prepared.media
+        image_texts = [
+            processor.replace_image_token(media, image_idx=i)
+            for i in range(len(prepared.images))
+        ]
+        expanded = [
+            processor.get_text_with_replacements(
+                [text], image_texts, prepared.video_texts
+            )[0][0]
+            for text in texts
+        ]
+        encoded = processor.tokenizer(
+            expanded, padding=True, return_token_type_ids=False, return_tensors="pt"
+        )
+        if encoded["input_ids"].shape[1] != width:
+            raise RuntimeError(
+                f"planned {width} input tokens, the processor produced {encoded['input_ids'].shape[1]}"
+            )
+        rows = len(texts)
+        ids = encoded["input_ids"].to(self.device)
+        mask = encoded["attention_mask"].to(self.device)
+        types = torch.as_tensor(
+            processor.create_mm_token_type_ids(encoded["input_ids"]), device=self.device
+        )
+        padded = not bool(encoded["attention_mask"].all())
+        with torch.inference_mode(), sdpa_backends(self.device):
+            embeds = backbone.get_input_embeddings()(ids)
+            image_grid = None
+            if "image" in features:
+                values = torch.cat(features["image"] * rows).to(
+                    embeds.device, embeds.dtype
+                )
+                image_mask, _ = backbone.get_placeholder_mask(
+                    ids, inputs_embeds=embeds, image_features=values
+                )
+                embeds = embeds.masked_scatter(image_mask, values)
+                image_grid = media["image_grid_thw"].repeat(rows, 1).to(self.device)
+            values = torch.cat(features["video"] * rows).to(embeds.device, embeds.dtype)
+            _, video_mask = backbone.get_placeholder_mask(
+                ids, inputs_embeds=embeds, video_features=values
+            )
+            embeds = embeds.masked_scatter(video_mask, values)
+            positions = backbone.compute_3d_position_ids(
+                input_ids=ids,
+                image_grid_thw=image_grid,
+                video_grid_thw=media["video_grid_thw"].repeat(rows, 1).to(self.device),
+                inputs_embeds=embeds,
+                attention_mask=mask,
+                past_key_values=None,
+                mm_token_type_ids=types,
+            )
+            fused = self.fast.fused if self.fast is not None else None
+            if fused is not None:
+                rotary = self.fast.lm.rotary_emb(embeds, positions)
+                hidden = fused.forward(
+                    embeds,
+                    mask.bool()[:, None, None, :],
+                    mask.reshape(-1) if padded else None,
+                    rotary,
+                )[:, -1]
+            else:
+                hidden = backbone.language_model(
+                    input_ids=None,
+                    position_ids=positions,
+                    attention_mask=mask,
+                    past_key_values=None,
+                    inputs_embeds=embeds,
+                    use_cache=False,
+                ).last_hidden_state[:, -1]
+            if self.readout_dtype == "float32":
+                logits = hidden.float() @ self.readout.T
+            else:
+                logits = torch.nn.functional.linear(hidden, self.readout).float()
+            limit = torch.as_tensor(list(counts), device=self.device)[:, None]
+            invalid = torch.arange(MAX_OPTIONS, device=self.device)[None] >= limit
+            return logits.masked_fill(invalid, float("-inf"))
+
+    def _run_videos(self, prepared: Prepared) -> tuple[dict[str, list[float]], int]:
+        keys = prepared.runnable
+        out: dict[str, list[float]] = {}
+        if not keys:
+            return out, 0
+        features = self.media_features(prepared.media)
+
+        def probabilities(texts, counts, width):
+            probs = (
+                (
+                    self.video_logits(texts, prepared, features, counts, width)
+                    / self.temperature
+                )
+                .softmax(-1)
+                .cpu()
+                .tolist()
+            )
+            return [p[:c] for p, c in zip(probs, counts)]
+
+        for start in range(0, len(keys), self.batch_size):
+            chunk = keys[start : start + self.batch_size]
+            extra = [k for k in chunk if k in prepared.reversed_texts]
+            texts = [prepared.texts[k] for k in chunk] + [
+                prepared.reversed_texts[k] for k in extra
+            ]
+            counts = [len(prepared.questions[k].keys) for k in chunk + extra]
+            widths = [prepared.lengths[k] for k in chunk] + [
+                prepared.reversed_lengths[k] for k in extra
+            ]
+            n = len(chunk)
+            if extra and len(widths) * max(widths) > MERGE_TOKENS:
+                probs = probabilities(
+                    texts[:n], counts[:n], max(widths[:n])
+                ) + probabilities(texts[n:], counts[n:], max(widths[n:]))
+            else:
+                probs = probabilities(texts, counts, max(widths))
+            out.update(zip(chunk, probs))
+            for key, backward in zip(extra, probs[n:]):
+                out[key] = average_orders(out[key], backward)
+        return out, sum(prepared.lengths[k] for k in keys) + sum(
+            prepared.reversed_lengths.values()
+        )
+
     def run(self, prepared: Prepared) -> tuple[dict[str, list[float]], int]:
         """Probabilities per runnable question (request order, ``batch_size`` per pass) and the input tokens."""
+        if prepared.videos:
+            return self._run_videos(prepared)
         if prepared.images:
             return self._run_images(prepared)
         keys = prepared.runnable
@@ -1255,15 +1923,17 @@ class D3:
         state: Any,
         questions: Mapping[str, Any],
         images: Sequence[Any] | None = None,
+        videos: Sequence[Any] | None = None,
     ) -> dict[str, Any]:
         """Typed Choice / Noul / Score answers about one state: ``{"model", "answers", "usage"}``.
 
         ``images``: any number of images (PIL images, paths, http(s) or data URLs) that every question sees.
+        ``videos``: any number of videos (paths, http(s) or data URLs, frame arrays) that every question sees.
         A question that cannot be answered gets ``{"type", "error", "message"}`` with ``error`` one of
         ``invalid_question``, ``max_length_exceeded`` (never truncated) or ``invalid_model_output``; the
         other questions of the request are still answered.
         """
-        prepared = self.prepare(state, questions, images)
+        prepared = self.prepare(state, questions, images, videos)
         probabilities, tokens = self.run(prepared)
         return self.respond(prepared, probabilities, tokens)
 
@@ -1359,6 +2029,7 @@ class D3:
             "over the question's codes only; argmax choice; no truncation (over-limit questions are "
             "refused); no option filtering; one fixed prompt for every request.",
             "images": self.image_contract(),
+            "videos": self.video_contract(),
         }
         if self.permutation_average:
             record["permutation_average"] = True
@@ -1389,6 +2060,31 @@ class D3:
             contract["unavailable"] = self.image_unavailable
         return contract
 
+    def video_contract(self) -> dict[str, Any]:
+        """How video inputs are read (or why they are not available)."""
+        contract: dict[str, Any] = {
+            "supported": self.video_unavailable is None,
+            "fps": VIDEO_FPS,
+            "min_frames": VIDEO_MIN_FRAMES,
+            "max_frames": VIDEO_MAX_FRAMES,
+            "max_pixels_per_frame": VIDEO_MAX_PIXELS,
+            "max_tokens_per_request": VIDEO_MAX_TOKENS,
+            "placement": "after the images, before the text of the user turn, one placeholder per video, "
+            "request order; every two frames are one token group after their timestamp",
+            "sampling": "source frames spread evenly over the whole video (frame arrays: frames at 2 per second)",
+        }
+        if self.video_unavailable is None:
+            contract["video_processor"] = type(self.processor.video_processor).__name__
+            try:
+                import cv2
+
+                contract["opencv"] = cv2.__version__
+            except Exception:  # noqa: BLE001
+                contract["opencv"] = None
+        else:
+            contract["unavailable"] = self.video_unavailable
+        return contract
+
     def runtime_info(self) -> dict[str, Any]:
         import transformers
 

d3_server.py

@@ -3,12 +3,16 @@
     pip install fastapi uvicorn
     python d3_server.py --model <package dir or Hub id> [--device cuda:0] [--host 127.0.0.1] [--port 8000]
 
-Request ``{"model", "state", "questions", "images"}``, response ``{"model", "answers", "usage"}``: the wire
-format of the Decision Index ``http`` engine. ``images`` (optional) lists any number of base64 PNG, JPEG or WebP
-data URLs (``data:image/png;base64,...``) that every question sees, each at most 8,000,000 bytes and 16,000,000
-pixels (the model reads it at up to 1.6 MP). A question over the input limit refuses the whole request with
-HTTP 422 naming the maximum context length (the Index records it as unsupported; nothing is truncated);
-malformed requests and invalid images also get 422. Requests are served one at a time. With
+Request ``{"model", "state", "questions", "images", "videos"}``, response ``{"model", "answers", "usage"}``: the
+wire format of the Decision Index ``http`` engine. ``images`` (optional) lists any number of base64 PNG, JPEG or
+WebP data URLs (``data:image/png;base64,...``) that every question sees, each at most 8,000,000 bytes and
+16,000,000 pixels (the model reads it at up to 1.6 MP). ``videos`` (optional) lists any number of base64 MP4, WebM,
+QuickTime or Matroska data URLs (``data:video/mp4;base64,...``) that every question sees, each at most 32,000,000
+bytes, 300 seconds and 8,294,400 pixels per frame (the model reads 2 frames per second, at most 32 frames spread
+over the video, each at up to 0.2 MP; at most 16,384 video tokens per request). A question over the input limit
+refuses the whole request with HTTP 422 naming the maximum context length (the Index records it as unsupported;
+nothing is truncated); malformed requests and invalid images or videos also get 422. Requests are served one at
+a time. With
 ``DECISION_API_KEY`` set, requests need ``Authorization: Bearer <key>``. ``GET /health`` and
 ``GET /v1/models`` describe the loaded model.
 
@@ -35,18 +39,28 @@ os.environ.setdefault("TRITON_CACHE_AUTOTUNING", "1")
 from d3_runtime import (  # noqa: E402
     DEFAULT_BATCH_SIZE,
     IMAGE_MAX_PIXELS,
+    MAX_VIDEO_BYTES,
+    VIDEO_FPS,
+    VIDEO_MAX_FRAMES,
+    VIDEO_MAX_PIXELS,
+    VIDEO_MAX_TOKENS,
     D3,
 )
 
-REQUEST_FIELDS = {"model", "state", "questions", "images"}
+REQUEST_FIELDS = {"model", "state", "questions", "images", "videos"}
 
 
 def reads_images(model: D3) -> bool:
     return getattr(model, "image_unavailable", "unknown") is None
 
 
+def reads_videos(model: D3) -> bool:
+    return getattr(model, "video_unavailable", "unknown") is None
+
+
 def modalities(model: D3) -> list[str]:
-    return ["text", "image"] if reads_images(model) else ["text"]
+    names = ["text", "image"] if reads_images(model) else ["text"]
+    return names + ["video"] if reads_videos(model) else names
 
 
 class Service:
@@ -129,6 +143,14 @@ def build_app(args: argparse.Namespace):
         }
         if reads_images(model):
             entry["image_max_pixels"] = IMAGE_MAX_PIXELS
+        if reads_videos(model):
+            entry["video"] = {
+                "fps": VIDEO_FPS,
+                "max_frames": VIDEO_MAX_FRAMES,
+                "max_pixels_per_frame": VIDEO_MAX_PIXELS,
+                "max_tokens_per_request": VIDEO_MAX_TOKENS,
+                "max_bytes": MAX_VIDEO_BYTES,
+            }
         return {"models": [entry]}
 
     def decode_images(model: D3, images: Any) -> list[Any]:
@@ -140,10 +162,25 @@ def build_app(args: argparse.Namespace):
             raise ValueError("This model reads text only; images are not supported.")
         return model.load_images(images, strict=True)
 
-    def decide(body: dict[str, Any], images: list[Any]) -> dict[str, Any]:
+    def decode_videos(model: D3, videos: Any) -> list[Any]:
+        if not isinstance(videos, list):
+            raise ValueError("videos must be a list of base64 data URLs")
+        if not videos:
+            return []
+        if not reads_videos(model):
+            raise ValueError("This model does not read videos.")
+        return model.load_videos(videos, strict=True)
+
+    def decide(
+        body: dict[str, Any], images: list[Any], videos: list[Any]
+    ) -> dict[str, Any]:
         model = service.model
         with service.lock:
-            if images:
+            if videos:
+                prepared = model.prepare(
+                    body.get("state"), body.get("questions"), images, videos
+                )
+            elif images:
                 prepared = model.prepare(
                     body.get("state"), body.get("questions"), images
                 )
@@ -181,7 +218,12 @@ def build_app(args: argparse.Namespace):
                 if body.get("images") is not None
                 else []
             )
-            return await run_in_threadpool(decide, body, images)
+            videos = (
+                await run_in_threadpool(decode_videos, service.model, body["videos"])
+                if body.get("videos") is not None
+                else []
+            )
+            return await run_in_threadpool(decide, body, images, videos)
         except ValueError as exc:
             raise HTTPException(422, str(exc)) from exc
 

modeling_d3.py

@@ -1,8 +1,8 @@
 """d3 model for 🤗 Transformers (``trust_remote_code=True``).
 
 ``AutoModel.from_pretrained(repo, trust_remote_code=True)`` loads the repository through its own runtime
-(``d3_runtime.py``) and returns a model with ``system_one(state=..., questions={...}, images=[...])``
-(any number of images per request). The
+(``d3_runtime.py``) and returns a model with ``system_one(state=..., questions={...}, images=[...], videos=[...])``
+(any number of images and videos per request). The
 repository is a standard ``Qwen3_5Model`` checkpoint (``D3Model``) or ``Qwen3VLModel`` checkpoint
 (``D3Qwen3VLModel``) plus a 255-way answer-code readout, so without ``trust_remote_code`` the same
 repository loads as the plain backbone. A directory without ``decision_config.json`` is not a Decision
@@ -107,7 +107,7 @@ def _is_decision(name_or_path: Any, revision: Any, hub: dict[str, Any]) -> bool:
 
 
 class D3Model(PreTrainedModel):
-    """A d3 checkpoint behind System One: ``system_one(state=..., questions={...}, images=[...])``."""
+    """A d3 checkpoint behind System One: ``system_one(state=..., questions={...}, images=[...], videos=[...])``."""
 
     config_class = Qwen3_5Config
     base_model_prefix = "decision"
@@ -233,16 +233,19 @@ class D3Model(PreTrainedModel):
         state: Any,
         questions: dict[str, Any],
         images: list[Any] | None = None,
+        videos: list[Any] | None = None,
     ) -> dict[str, Any]:
         """Typed Choice / Noul / Score answers about one state: ``{"model", "answers", "usage"}``.
 
         ``questions`` maps question IDs to ``{"type": "choice" | "noul" | "score", "instructions": ...,
         "criteria": ...}``; a question over the input limit is answered ``max_length_exceeded``, never
         truncated. ``images``: any number of images every question sees (PIL images, local paths, http(s) URLs or
-        base64 ``data:image/...`` URLs), placed before the text and read at up to 1.6 MP each.
+        base64 ``data:image/...`` URLs), placed before the text and read at up to 1.6 MP each. ``videos``: any
+        number of videos every question sees (local paths, http(s) URLs, base64 ``data:video/...`` URLs or frame
+        arrays), placed after the images: 2 frames per second, at most 32 frames, each at up to 0.2 MP.
         """
         return self._require().system_one(
-            state=state, questions=questions, images=images
+            state=state, questions=questions, images=images, videos=videos
         )
 
     def forward(
@@ -250,8 +253,11 @@ class D3Model(PreTrainedModel):
         state: Any = None,
         questions: dict[str, Any] | None = None,
         images: list[Any] | None = None,
+        videos: list[Any] | None = None,
     ) -> dict[str, Any]:
-        return self.system_one(state=state, questions=questions, images=images)
+        return self.system_one(
+            state=state, questions=questions, images=images, videos=videos
+        )
 
     def to(self, *args: Any, **kwargs: Any) -> D3Model:
         """Move to another device; numerics are fixed by the checkpoint, so dtype casts are refused."""

pipeline_d3.py

@@ -1,8 +1,9 @@
 """The ``decision`` pipeline for d3 models (``trust_remote_code=True``).
 
 ``pipeline("decision", model=repo, trust_remote_code=True)`` loads the model with ``AutoModel`` and answers
-``{"state": ..., "questions": {...}}`` requests, optionally with ``"images": [...]`` (any number of images per
-request), or ``state=..., questions=..., images=...`` keywords, or a list of requests, with the model's
+``{"state": ..., "questions": {...}}`` requests, optionally with ``"images": [...]`` and ``"videos": [...]`` (any
+number per request), or ``state=..., questions=..., images=..., videos=...`` keywords, or a list of requests, with
+the model's
 ``system_one`` response. The model batches the questions of one request itself.
 """
 
@@ -10,7 +11,7 @@ from transformers import Pipeline
 
 _UNSET = object()
 REQUEST_KEYS = {"state", "questions"}
-OPTIONAL_KEYS = {"images"}
+OPTIONAL_KEYS = {"images", "videos"}
 
 
 class D3Pipeline(Pipeline):
@@ -28,17 +29,28 @@ class D3Pipeline(Pipeline):
         return {}, {}, {}
 
     def __call__(
-        self, inputs=None, *, state=_UNSET, questions=_UNSET, images=_UNSET, **kwargs
+        self,
+        inputs=None,
+        *,
+        state=_UNSET,
+        questions=_UNSET,
+        images=_UNSET,
+        videos=_UNSET,
+        **kwargs,
     ):
-        if state is not _UNSET or questions is not _UNSET or images is not _UNSET:
+        if any(v is not _UNSET for v in (state, questions, images, videos)):
             if inputs is not None:
-                raise TypeError("Pass one request, or state=, questions= and images=")
+                raise TypeError(
+                    "Pass one request, or state=, questions=, images= and videos="
+                )
             inputs = {
                 "state": None if state is _UNSET else state,
                 "questions": None if questions is _UNSET else questions,
             }
             if images is not _UNSET:
                 inputs["images"] = images
+            if videos is not _UNSET:
+                inputs["videos"] = videos
         if kwargs.get("batch_size") not in (None, 1):
             raise ValueError(
                 "The decision pipeline runs one request at a time (batch_size=1)"
@@ -51,12 +63,13 @@ class D3Pipeline(Pipeline):
         ):
             raise ValueError(
                 'A decision request is {"state": ..., "questions": {<id>: <question>, ...}} '
-                'with optional "images": [...]'
+                'with optional "images": [...] and "videos": [...]'
             )
         return {
             "state": inputs["state"],
             "questions": inputs["questions"],
             "images": inputs.get("images"),
+            "videos": inputs.get("videos"),
         }
 
     def _forward(self, model_inputs):
@@ -64,6 +77,7 @@ class D3Pipeline(Pipeline):
             state=model_inputs["state"],
             questions=model_inputs["questions"],
             images=model_inputs["images"],
+            videos=model_inputs["videos"],
         )
 
     def postprocess(self, model_outputs):

requirements.txt

@@ -1,4 +1,4 @@
-# d3 runtime. Validated with transformers 5.17.0 on ROCm (torch 2.13) and CUDA (RTX PRO 6000).
+# d3 runtime. Validated with transformers 5.17.0 on ROCm (torch 2.13).
 torch>=2.8
 transformers==5.17.0
 accelerate>=1.10
@@ -11,7 +11,8 @@ einops
 # (install the torchvision build that matches torch; without it transformers falls back to a PIL resize).
 pillow>=10
 torchvision
-# Optional on CUDA: causal-conv1d (the convolution then runs in its CUDA kernel instead of torch conv1d).
+# Video files: OpenCV decodes them with FFmpeg (frame arrays need no decoder).
+opencv-python-headless
 # d3_server.py only:
 # fastapi
 # uvicorn