"""Encoder-feature plumbing for the NASA-IBM Lunar Foundation Model explorer. Model / data loading mirrors the authors' reference notebook (`examples/generate_images.ipynb` in NASA-IMPACT/NASA-IBM-Lunar-Foundation-Model): same `config.yaml`, modality info, dataset transforms and checkpoint loader. Only the *encoder* half is run here: raw modalities are patch-embedded (optionally at a FlexiViT runtime patch size), concatenated with the register / text tokens, passed through the 12-layer ViT-B encoder, and the per-patch output tokens are returned for analysis. Kept free of `gradio` imports so it can be exercised head-less. """ from __future__ import annotations import os import re import threading from functools import lru_cache from pathlib import Path import numpy as np import torch from huggingface_hub import snapshot_download from matplotlib import colormaps from omegaconf import OmegaConf from tokenizers import Tokenizer # `ni_lfm` resolves the text tokenizer and metadata binning table relative to # the repo root, exactly as the reference notebook does with `os.chdir`. APP_DIR = Path(__file__).resolve().parent os.chdir(APP_DIR) from ni_lfm.data.modality_info import setup_modality_transform # noqa: E402 from ni_lfm.models.generate import init_full_input_modality # noqa: E402 from ni_lfm.models.ni_lfm_generation import get_lfm_generation_model # noqa: E402 from ni_lfm.utils import generation as G # noqa: E402 MODEL_REPO = "nasa-ibm-ai4science/NASA-IBM-Lunar-Foundation-Model" DATA_REPO = "nasa-ibm-ai4science/Sombench-pretraining-data" TEXT_MODALITIES = ("metadata", "static_maps") PATCH_SIZES = (16, 8) # 16 = pretraining; 8 = FlexiViT, used for the model card's results TRACKS: dict[str, dict] = { "WAC": { "label": "WAC — regional, 51.2 km tiles", "folder": "WAC_LowRes", "parquet": "WAC_LowRes/WAC_LowRes.parquet", "image_modalities": ["vis", "uv", "dtm", "slope", "aspect"], "main": "vis", }, "NAC": { "label": "NAC — local, 512 m tiles", "folder": "NAC_HighRes", "parquet": "NAC_HighRes/NAC_HighRes.parquet", "image_modalities": ["nac", "dtm_3m", "slope_3m", "aspect_3m"], "main": "nac", }, } MOD_META: dict[str, dict] = { "vis": {"title": "WAC VIS reflectance", "gsd": "100 m"}, "uv": {"title": "WAC UV reflectance", "gsd": "500 m"}, "dtm": {"title": "Elevation (DTM)", "gsd": "60 m"}, "slope": {"title": "Slope", "gsd": "60 m"}, "aspect": {"title": "Aspect (sin, cos)", "gsd": "60 m"}, "nac": {"title": "NAC reflectance", "gsd": "1 m"}, "dtm_3m": {"title": "Elevation (NAC DTM)", "gsd": "3 m"}, "slope_3m": {"title": "Slope", "gsd": "3 m"}, "aspect_3m": {"title": "Aspect (sin, cos)", "gsd": "3 m"}, "metadata": {"title": "Acquisition geometry (text tokens)", "gsd": "—"}, "static_maps": {"title": "Static map priors (text tokens)", "gsd": "—"}, } _MODEL_LOCK = threading.Lock() # --------------------------------------------------------------------------- # # Startup: weights, config, data # --------------------------------------------------------------------------- # def _load_config(path: Path): """Load the pretraining config, repairing a duplicated key in the release. `backbone/config.yaml` ships `text_tokenizer_path: text_tokenizer_path: ...`, which is not valid YAML, so the line is patched before parsing. """ text = path.read_text() text = re.sub( r"^(\s*text_tokenizer_path:).*$", r"\1 ni_lfm/utils/tokenizer/trained/text_tokenizer.json", text, count=1, flags=re.M, ) return OmegaConf.create(text) print("Downloading backbone weights ...", flush=True) WEIGHTS_ROOT = Path( os.environ.get("LFM_WEIGHTS_DIR") or snapshot_download(MODEL_REPO, allow_patterns=["backbone/*"]) ) print("Downloading SomBench sample tiles ...", flush=True) DATA_ROOT = Path( os.environ.get("LFM_DATA_DIR") or snapshot_download(DATA_REPO, repo_type="dataset", allow_patterns=["WAC_LowRes/**", "NAC_HighRes/**"]) ) CFG = _load_config(WEIGHTS_ROOT / "backbone" / "config.yaml") MODALITY_INFO, INPUT_SIZE = G.build_modality_info(CFG) TEXT_TOKENIZER = Tokenizer.from_file(CFG.data.text_tokenizer_path) IN_DOMAINS = list(CFG.data.in_domains) OUT_DOMAINS = list(CFG.data.out_domains) ALL_DOMAINS = sorted(set(IN_DOMAINS) | set(OUT_DOMAINS)) MODEL_MOD_INFO = G.filter_modality_info(ALL_DOMAINS, MODALITY_INFO) INPUT_MODALITIES: dict[str, list[str]] = { track: [*spec["image_modalities"], *TEXT_MODALITIES] for track, spec in TRACKS.items() } def _build_datasets() -> dict[str, object]: datasets = {} for track, spec in TRACKS.items(): mods = INPUT_MODALITIES[track] mod_info = G.filter_modality_info(mods, MODALITY_INFO) datasets[track] = G.LunarGenerationDataset( data_root=str(DATA_ROOT / spec["folder"]), index_path=str(DATA_ROOT / spec["parquet"]), modalities=mods, modality_transforms=setup_modality_transform(domains=mods, modality_info=mod_info), center_crop=G.CenterCropImage(target_size=INPUT_SIZE, main_domain=spec["main"]), max_samples=None, remove_key_mod_nan=True, return_file_name=True, ) return datasets DATASETS = _build_datasets() def _build_catalog() -> dict[str, dict]: """`":"` -> tile description, for every sample tile.""" catalog = {} for track, ds in DATASETS.items(): for i in range(len(ds)): r = ds.index.iloc[i] lat, lon = float(r["CENTER_LATITUDE"]), float(r["CENTER_LONGITUDE"]) name = f"{r['PRODUCT_ID']}_r{int(r['ROW'])}_c{int(r['COL'])}" catalog[f"{track}:{i}"] = { "track": track, "idx": i, "name": name, "row": r, "label": f"{track} · {name} · lat {lat:+.2f}°, lon {lon:+.2f}° · " f"inc {float(r['INCIDENCE_ANGLE']):.0f}°", } return catalog CATALOG = _build_catalog() print(f"Sample tiles: {len(CATALOG)} ({', '.join(f'{t}={len(d)}' for t, d in DATASETS.items())})", flush=True) def tile_choices() -> list[tuple[str, str]]: return [(t["label"], key) for key, t in CATALOG.items()] def tile_track(tile: str) -> str: return CATALOG[tile]["track"] def tile_info_md(tile: str) -> str: t = CATALOG[tile] r = t["row"] return ( f"**{TRACKS[t['track']]['label']}** · `{t['name']}` · centre " f"**{float(r['CENTER_LATITUDE']):+.3f}°, {float(r['CENTER_LONGITUDE']):+.3f}°**\n\n" f"incidence **{float(r['INCIDENCE_ANGLE']):.1f}°** · emission **{float(r['EMISSION_ANGLE']):.1f}°** · " f"phase **{float(r['PHASE_ANGLE']):.1f}°** · sun azimuth **{float(r['SUB_SOLAR_GROUND_AZIMUTH']):.0f}°**" ) # --------------------------------------------------------------------------- # # Model — encoder only, so no VQ tokenizers / diffusion decoders are loaded. # --------------------------------------------------------------------------- # print(f"Building model: {CFG.model.name}", flush=True) MODEL = get_lfm_generation_model( variant=CFG.model.name, input_modalities=IN_DOMAINS, output_modalities=OUT_DOMAINS, modality_info=MODEL_MOD_INFO, pretrained_tokenizers=torch.nn.ModuleDict(), text_tokenizer=TEXT_TOKENIZER, cfg=CFG, num_register_tokens=CFG.model.num_register_tokens, ) G.load_ckpt_for_generation(str(WEIGHTS_ROOT / "backbone" / "checkpoint.pt"), MODEL) MODEL.eval() DEVICE = "cuda" if torch.cuda.is_available() else "cpu" MODEL.to(DEVICE) SAMPLER = MODEL.sampler ENCDEC = SAMPLER.model print(f"Model parameters: {sum(p.numel() for p in MODEL.parameters()):,}", flush=True) @lru_cache(maxsize=32) def load_tile(tile: str) -> dict: t = CATALOG[tile] return G.collate_fn([DATASETS[t["track"]][t["idx"]]]) def _to_chw(x) -> np.ndarray: a = x.detach().cpu().float().numpy() if torch.is_tensor(x) else np.asarray(x) return a[0] if a.ndim == 4 else (a[None] if a.ndim == 2 else a) # --------------------------------------------------------------------------- # # Encoder forward # --------------------------------------------------------------------------- # @torch.no_grad() def encode(tile: str, modalities: list[str], patch_size: int) -> dict[str, np.ndarray]: """Run the encoder on one tile and return per-patch output tokens. Returns `{image_modality: (grid*grid, 768) float32}`; text modalities are fed to the encoder (they condition every patch through attention) but have no spatial layout, so they are not returned. """ track = tile_track(tile) mods = [m for m in modalities if m in INPUT_MODALITIES[track]] img_mods = [m for m in mods if m not in TEXT_MODALITIES] if not img_mods: raise ValueError("Select at least one image modality of this tile's track.") if patch_size not in PATCH_SIZES: raise ValueError(f"patch_size must be one of {PATCH_SIZES}") batch = load_tile(tile) inputs = G.prepare_model_inputs( batch=batch, in_domains=mods, modality_info=MODALITY_INFO, text_tokenizer=TEXT_TOKENIZER, device=DEVICE, ) mod_dict = {} for mod, value in inputs.items(): if MODALITY_INFO[mod]["type"] == "img": h, w = value.shape[-2:] d = {"tensor": value, "runtime_patch_size": int(patch_size)} mod_dict[mod] = init_full_input_modality( d, MODALITY_INFO, mod, DEVICE, eos_id=MODEL.eos_id, num_tokens=(h // patch_size) * (w // patch_size), ) else: mod_dict[mod] = init_full_input_modality(value, MODALITY_INFO, mod, DEVICE, eos_id=MODEL.eos_id) with _MODEL_LOCK: enc = {m: ENCDEC.encoder_embeddings[m](d) for m, d in mod_dict.items()} tokens, emb, mask, mod_mask = SAMPLER.forward_mask_encoder_generation(enc) x = ENCDEC.forward_encoder(tokens + emb, mask) x, mod_mask = x[0].float().cpu(), mod_mask[0].cpu() out = {} for mod in img_mods: feats = x[mod_mask == MODALITY_INFO[mod]["id"]] expected = enc[mod]["x"].shape[1] if feats.shape[0] != expected: raise RuntimeError(f"{mod}: got {feats.shape[0]} encoder tokens, expected {expected}") out[mod] = feats.numpy().astype(np.float32) return out # --------------------------------------------------------------------------- # # Analysis (CPU, numpy) # --------------------------------------------------------------------------- # def _l2n(a: np.ndarray) -> np.ndarray: return a / (np.linalg.norm(a, axis=-1, keepdims=True) + 1e-8) def fuse(feats: dict[str, np.ndarray]) -> np.ndarray: """Per-location fused embedding: mean of the L2-normalised modality tokens.""" return np.mean([_l2n(f) for f in feats.values()], axis=0) def pca_rgb(feats: dict[str, np.ndarray], fused: np.ndarray) -> tuple[np.ndarray, dict[str, np.ndarray], np.ndarray]: """Fit one 3-component PCA basis on every token, so colours are comparable across modalities and with the fused map. Returns (fused_rgb, per_mod_rgb, explained).""" pool = np.concatenate([_l2n(f) for f in feats.values()] + [fused], axis=0) mean = pool.mean(0, keepdims=True) _, s, vt = np.linalg.svd(pool - mean, full_matrices=False) basis = vt[:3].T explained = (s[:3] ** 2) / (s ** 2).sum() proj_pool = (pool - mean) @ basis lo = np.percentile(proj_pool, 2, axis=0) hi = np.percentile(proj_pool, 98, axis=0) def to_rgb(f): p = ((f - mean) @ basis - lo) / (hi - lo + 1e-8) return np.clip(p, 0, 1) return to_rgb(fused), {m: to_rgb(_l2n(f)) for m, f in feats.items()}, explained def kmeans(x: np.ndarray, k: int, iters: int = 50, seed: int = 0) -> np.ndarray: """Cosine k-means (k-means++ init) on L2-normalised features.""" x = _l2n(x) rng = np.random.default_rng(seed) centers = [x[rng.integers(len(x))]] for _ in range(1, k): d = 1.0 - np.max(x @ np.stack(centers).T, axis=1) p = np.clip(d, 0, None) ** 2 centers.append(x[rng.choice(len(x), p=p / p.sum())] if p.sum() > 0 else x[rng.integers(len(x))]) c = np.stack(centers) labels = np.zeros(len(x), dtype=int) for it in range(iters): new = np.argmax(x @ c.T, axis=1) if it and np.array_equal(new, labels): break labels = new for j in range(k): members = x[labels == j] if len(members): c[j] = _l2n(members.mean(0)) # Relabel by cluster size so colours are stable-ish across runs. order = np.argsort(-np.bincount(labels, minlength=k)) remap = np.empty(k, dtype=int) remap[order] = np.arange(k) return remap[labels] def cross_modal_alignment(feats: dict[str, np.ndarray]) -> list[tuple[str, str, float, float]]: """For each modality pair: mean cosine of same-location tokens vs. different-location tokens.""" mods = list(feats) rows = [] for i in range(len(mods)): for j in range(i + 1, len(mods)): a, b = _l2n(feats[mods[i]]), _l2n(feats[mods[j]]) sim = a @ b.T n = len(sim) diag = float(np.trace(sim) / n) off = float((sim.sum() - np.trace(sim)) / (n * n - n)) rows.append((mods[i], mods[j], diag, off)) return rows # --------------------------------------------------------------------------- # # Rendering # --------------------------------------------------------------------------- # def _cmap(name: str, x01: np.ndarray) -> np.ndarray: return (colormaps[name](np.clip(x01, 0, 1))[..., :3] * 255).round().astype(np.uint8) def _stretch(a: np.ndarray) -> np.ndarray: lo, hi = np.nanpercentile(a, 1), np.nanpercentile(a, 99) return (a - lo) / (hi - lo + 1e-8) def render_modality(tile: str, mod: str) -> np.ndarray: """Render one image modality of a tile as RGB uint8 (H, W, 3).""" a = _to_chw(G.denormalize_data(load_tile(tile)[mod], mod, MODALITY_INFO)) if mod == "vis": # [415, 566, 604, 643, 689] nm -> natural-ish RGB rgb = np.stack([a[3], a[1], a[0]], axis=-1) return (np.clip(_stretch(rgb), 0, 1) * 255).round().astype(np.uint8) if mod in ("uv", "nac"): return _cmap("gray", _stretch(a[0])) if mod.startswith("dtm"): return _cmap("terrain", _stretch(a[0])) if mod.startswith("slope"): return _cmap("magma", _stretch(a[0])) if mod.startswith("aspect"): return _cmap("twilight", (np.degrees(np.arctan2(a[0], a[1])) % 360) / 360) return _cmap("viridis", _stretch(a[0])) def upsample(grid_img: np.ndarray, size: int) -> np.ndarray: """Nearest-neighbour upsample a (g, g[, C]) map to (size, size[, C]).""" rep = size // grid_img.shape[0] return np.repeat(np.repeat(grid_img, rep, axis=0), rep, axis=1) def grid_rgb(flat: np.ndarray, size: int) -> np.ndarray: g = int(round(np.sqrt(flat.shape[0]))) img = flat.reshape(g, g, -1) if img.dtype != np.uint8: img = (np.clip(img, 0, 1) * 255).round().astype(np.uint8) return upsample(img, size) def blend(base: np.ndarray, overlay: np.ndarray, alpha: float) -> np.ndarray: gray = base.astype(np.float32).mean(-1, keepdims=True).repeat(3, -1) return ((1 - alpha) * gray + alpha * overlay.astype(np.float32)).round().astype(np.uint8) def cluster_image(labels: np.ndarray, base: np.ndarray) -> np.ndarray: k = int(labels.max()) + 1 cmap = colormaps["tab20" if k > 10 else "tab10"] colors = (np.array([cmap(i)[:3] for i in range(k)]) * 255).astype(np.uint8) return blend(base, grid_rgb(colors[labels], base.shape[0]), 0.6) def similarity_image(fused: np.ndarray, patch: int, base: np.ndarray) -> tuple[np.ndarray, np.ndarray]: f = _l2n(fused) sim = f @ f[patch] lo = np.percentile(sim, 1) heat = _cmap("inferno", (sim - lo) / (1 - lo + 1e-8)) img = blend(base, grid_rgb(heat, base.shape[0]), 0.75) g = int(round(np.sqrt(len(sim)))) cell = base.shape[0] // g r, c = divmod(patch, g) y0, x0 = r * cell, c * cell img[y0:y0 + cell, [x0, min(x0 + cell - 1, img.shape[1] - 1)]] = (0, 255, 255) img[[y0, min(y0 + cell - 1, img.shape[0] - 1)], x0:x0 + cell] = (0, 255, 255) return img, sim