Spaces:
Running on Zero
Running on Zero
Download lfm_features.py from Onyoin/lunar-fm-feature-explorer: direct link, hf CLI and curl.
- Browser
- Download file 16.5 kB
-
https://huggingface.co/spaces/Onyoin/lunar-fm-feature-explorer/resolve/main/lfm_features.py
- Command line
-
hf download hf://spaces/Onyoin/lunar-fm-feature-explorer/lfm_features.py
-
curl -L -o lfm_features.py https://huggingface.co/spaces/Onyoin/lunar-fm-feature-explorer/resolve/main/lfm_features.py
16.5 kB
| """Encoder-feature plumbing for the NASA-IBM Lunar Foundation Model explorer. | |
| Model / data loading mirrors the authors' reference notebook | |
| (`examples/generate_images.ipynb` in NASA-IMPACT/NASA-IBM-Lunar-Foundation-Model): | |
| same `config.yaml`, modality info, dataset transforms and checkpoint loader. | |
| Only the *encoder* half is run here: raw modalities are patch-embedded | |
| (optionally at a FlexiViT runtime patch size), concatenated with the | |
| register / text tokens, passed through the 12-layer ViT-B encoder, and the | |
| per-patch output tokens are returned for analysis. | |
| Kept free of `gradio` imports so it can be exercised head-less. | |
| """ | |
| from __future__ import annotations | |
| import os | |
| import re | |
| import threading | |
| from functools import lru_cache | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| from huggingface_hub import snapshot_download | |
| from matplotlib import colormaps | |
| from omegaconf import OmegaConf | |
| from tokenizers import Tokenizer | |
| # `ni_lfm` resolves the text tokenizer and metadata binning table relative to | |
| # the repo root, exactly as the reference notebook does with `os.chdir`. | |
| APP_DIR = Path(__file__).resolve().parent | |
| os.chdir(APP_DIR) | |
| from ni_lfm.data.modality_info import setup_modality_transform # noqa: E402 | |
| from ni_lfm.models.generate import init_full_input_modality # noqa: E402 | |
| from ni_lfm.models.ni_lfm_generation import get_lfm_generation_model # noqa: E402 | |
| from ni_lfm.utils import generation as G # noqa: E402 | |
| MODEL_REPO = "nasa-ibm-ai4science/NASA-IBM-Lunar-Foundation-Model" | |
| DATA_REPO = "nasa-ibm-ai4science/Sombench-pretraining-data" | |
| TEXT_MODALITIES = ("metadata", "static_maps") | |
| PATCH_SIZES = (16, 8) # 16 = pretraining; 8 = FlexiViT, used for the model card's results | |
| TRACKS: dict[str, dict] = { | |
| "WAC": { | |
| "label": "WAC — regional, 51.2 km tiles", | |
| "folder": "WAC_LowRes", | |
| "parquet": "WAC_LowRes/WAC_LowRes.parquet", | |
| "image_modalities": ["vis", "uv", "dtm", "slope", "aspect"], | |
| "main": "vis", | |
| }, | |
| "NAC": { | |
| "label": "NAC — local, 512 m tiles", | |
| "folder": "NAC_HighRes", | |
| "parquet": "NAC_HighRes/NAC_HighRes.parquet", | |
| "image_modalities": ["nac", "dtm_3m", "slope_3m", "aspect_3m"], | |
| "main": "nac", | |
| }, | |
| } | |
| MOD_META: dict[str, dict] = { | |
| "vis": {"title": "WAC VIS reflectance", "gsd": "100 m"}, | |
| "uv": {"title": "WAC UV reflectance", "gsd": "500 m"}, | |
| "dtm": {"title": "Elevation (DTM)", "gsd": "60 m"}, | |
| "slope": {"title": "Slope", "gsd": "60 m"}, | |
| "aspect": {"title": "Aspect (sin, cos)", "gsd": "60 m"}, | |
| "nac": {"title": "NAC reflectance", "gsd": "1 m"}, | |
| "dtm_3m": {"title": "Elevation (NAC DTM)", "gsd": "3 m"}, | |
| "slope_3m": {"title": "Slope", "gsd": "3 m"}, | |
| "aspect_3m": {"title": "Aspect (sin, cos)", "gsd": "3 m"}, | |
| "metadata": {"title": "Acquisition geometry (text tokens)", "gsd": "—"}, | |
| "static_maps": {"title": "Static map priors (text tokens)", "gsd": "—"}, | |
| } | |
| _MODEL_LOCK = threading.Lock() | |
| # --------------------------------------------------------------------------- # | |
| # Startup: weights, config, data | |
| # --------------------------------------------------------------------------- # | |
| def _load_config(path: Path): | |
| """Load the pretraining config, repairing a duplicated key in the release. | |
| `backbone/config.yaml` ships `text_tokenizer_path: text_tokenizer_path: ...`, | |
| which is not valid YAML, so the line is patched before parsing. | |
| """ | |
| text = path.read_text() | |
| text = re.sub( | |
| r"^(\s*text_tokenizer_path:).*$", | |
| r"\1 ni_lfm/utils/tokenizer/trained/text_tokenizer.json", | |
| text, | |
| count=1, | |
| flags=re.M, | |
| ) | |
| return OmegaConf.create(text) | |
| print("Downloading backbone weights ...", flush=True) | |
| WEIGHTS_ROOT = Path( | |
| os.environ.get("LFM_WEIGHTS_DIR") | |
| or snapshot_download(MODEL_REPO, allow_patterns=["backbone/*"]) | |
| ) | |
| print("Downloading SomBench sample tiles ...", flush=True) | |
| DATA_ROOT = Path( | |
| os.environ.get("LFM_DATA_DIR") | |
| or snapshot_download(DATA_REPO, repo_type="dataset", allow_patterns=["WAC_LowRes/**", "NAC_HighRes/**"]) | |
| ) | |
| CFG = _load_config(WEIGHTS_ROOT / "backbone" / "config.yaml") | |
| MODALITY_INFO, INPUT_SIZE = G.build_modality_info(CFG) | |
| TEXT_TOKENIZER = Tokenizer.from_file(CFG.data.text_tokenizer_path) | |
| IN_DOMAINS = list(CFG.data.in_domains) | |
| OUT_DOMAINS = list(CFG.data.out_domains) | |
| ALL_DOMAINS = sorted(set(IN_DOMAINS) | set(OUT_DOMAINS)) | |
| MODEL_MOD_INFO = G.filter_modality_info(ALL_DOMAINS, MODALITY_INFO) | |
| INPUT_MODALITIES: dict[str, list[str]] = { | |
| track: [*spec["image_modalities"], *TEXT_MODALITIES] for track, spec in TRACKS.items() | |
| } | |
| def _build_datasets() -> dict[str, object]: | |
| datasets = {} | |
| for track, spec in TRACKS.items(): | |
| mods = INPUT_MODALITIES[track] | |
| mod_info = G.filter_modality_info(mods, MODALITY_INFO) | |
| datasets[track] = G.LunarGenerationDataset( | |
| data_root=str(DATA_ROOT / spec["folder"]), | |
| index_path=str(DATA_ROOT / spec["parquet"]), | |
| modalities=mods, | |
| modality_transforms=setup_modality_transform(domains=mods, modality_info=mod_info), | |
| center_crop=G.CenterCropImage(target_size=INPUT_SIZE, main_domain=spec["main"]), | |
| max_samples=None, | |
| remove_key_mod_nan=True, | |
| return_file_name=True, | |
| ) | |
| return datasets | |
| DATASETS = _build_datasets() | |
| def _build_catalog() -> dict[str, dict]: | |
| """`"<TRACK>:<idx>"` -> tile description, for every sample tile.""" | |
| catalog = {} | |
| for track, ds in DATASETS.items(): | |
| for i in range(len(ds)): | |
| r = ds.index.iloc[i] | |
| lat, lon = float(r["CENTER_LATITUDE"]), float(r["CENTER_LONGITUDE"]) | |
| name = f"{r['PRODUCT_ID']}_r{int(r['ROW'])}_c{int(r['COL'])}" | |
| catalog[f"{track}:{i}"] = { | |
| "track": track, | |
| "idx": i, | |
| "name": name, | |
| "row": r, | |
| "label": f"{track} · {name} · lat {lat:+.2f}°, lon {lon:+.2f}° · " | |
| f"inc {float(r['INCIDENCE_ANGLE']):.0f}°", | |
| } | |
| return catalog | |
| CATALOG = _build_catalog() | |
| print(f"Sample tiles: {len(CATALOG)} ({', '.join(f'{t}={len(d)}' for t, d in DATASETS.items())})", flush=True) | |
| def tile_choices() -> list[tuple[str, str]]: | |
| return [(t["label"], key) for key, t in CATALOG.items()] | |
| def tile_track(tile: str) -> str: | |
| return CATALOG[tile]["track"] | |
| def tile_info_md(tile: str) -> str: | |
| t = CATALOG[tile] | |
| r = t["row"] | |
| return ( | |
| f"**{TRACKS[t['track']]['label']}** · `{t['name']}` · centre " | |
| f"**{float(r['CENTER_LATITUDE']):+.3f}°, {float(r['CENTER_LONGITUDE']):+.3f}°**\n\n" | |
| f"incidence **{float(r['INCIDENCE_ANGLE']):.1f}°** · emission **{float(r['EMISSION_ANGLE']):.1f}°** · " | |
| f"phase **{float(r['PHASE_ANGLE']):.1f}°** · sun azimuth **{float(r['SUB_SOLAR_GROUND_AZIMUTH']):.0f}°**" | |
| ) | |
| # --------------------------------------------------------------------------- # | |
| # Model — encoder only, so no VQ tokenizers / diffusion decoders are loaded. | |
| # --------------------------------------------------------------------------- # | |
| print(f"Building model: {CFG.model.name}", flush=True) | |
| MODEL = get_lfm_generation_model( | |
| variant=CFG.model.name, | |
| input_modalities=IN_DOMAINS, | |
| output_modalities=OUT_DOMAINS, | |
| modality_info=MODEL_MOD_INFO, | |
| pretrained_tokenizers=torch.nn.ModuleDict(), | |
| text_tokenizer=TEXT_TOKENIZER, | |
| cfg=CFG, | |
| num_register_tokens=CFG.model.num_register_tokens, | |
| ) | |
| G.load_ckpt_for_generation(str(WEIGHTS_ROOT / "backbone" / "checkpoint.pt"), MODEL) | |
| MODEL.eval() | |
| DEVICE = "cuda" if torch.cuda.is_available() else "cpu" | |
| MODEL.to(DEVICE) | |
| SAMPLER = MODEL.sampler | |
| ENCDEC = SAMPLER.model | |
| print(f"Model parameters: {sum(p.numel() for p in MODEL.parameters()):,}", flush=True) | |
| def load_tile(tile: str) -> dict: | |
| t = CATALOG[tile] | |
| return G.collate_fn([DATASETS[t["track"]][t["idx"]]]) | |
| def _to_chw(x) -> np.ndarray: | |
| a = x.detach().cpu().float().numpy() if torch.is_tensor(x) else np.asarray(x) | |
| return a[0] if a.ndim == 4 else (a[None] if a.ndim == 2 else a) | |
| # --------------------------------------------------------------------------- # | |
| # Encoder forward | |
| # --------------------------------------------------------------------------- # | |
| def encode(tile: str, modalities: list[str], patch_size: int) -> dict[str, np.ndarray]: | |
| """Run the encoder on one tile and return per-patch output tokens. | |
| Returns `{image_modality: (grid*grid, 768) float32}`; text modalities are fed | |
| to the encoder (they condition every patch through attention) but have no | |
| spatial layout, so they are not returned. | |
| """ | |
| track = tile_track(tile) | |
| mods = [m for m in modalities if m in INPUT_MODALITIES[track]] | |
| img_mods = [m for m in mods if m not in TEXT_MODALITIES] | |
| if not img_mods: | |
| raise ValueError("Select at least one image modality of this tile's track.") | |
| if patch_size not in PATCH_SIZES: | |
| raise ValueError(f"patch_size must be one of {PATCH_SIZES}") | |
| batch = load_tile(tile) | |
| inputs = G.prepare_model_inputs( | |
| batch=batch, in_domains=mods, modality_info=MODALITY_INFO, | |
| text_tokenizer=TEXT_TOKENIZER, device=DEVICE, | |
| ) | |
| mod_dict = {} | |
| for mod, value in inputs.items(): | |
| if MODALITY_INFO[mod]["type"] == "img": | |
| h, w = value.shape[-2:] | |
| d = {"tensor": value, "runtime_patch_size": int(patch_size)} | |
| mod_dict[mod] = init_full_input_modality( | |
| d, MODALITY_INFO, mod, DEVICE, eos_id=MODEL.eos_id, | |
| num_tokens=(h // patch_size) * (w // patch_size), | |
| ) | |
| else: | |
| mod_dict[mod] = init_full_input_modality(value, MODALITY_INFO, mod, DEVICE, eos_id=MODEL.eos_id) | |
| with _MODEL_LOCK: | |
| enc = {m: ENCDEC.encoder_embeddings[m](d) for m, d in mod_dict.items()} | |
| tokens, emb, mask, mod_mask = SAMPLER.forward_mask_encoder_generation(enc) | |
| x = ENCDEC.forward_encoder(tokens + emb, mask) | |
| x, mod_mask = x[0].float().cpu(), mod_mask[0].cpu() | |
| out = {} | |
| for mod in img_mods: | |
| feats = x[mod_mask == MODALITY_INFO[mod]["id"]] | |
| expected = enc[mod]["x"].shape[1] | |
| if feats.shape[0] != expected: | |
| raise RuntimeError(f"{mod}: got {feats.shape[0]} encoder tokens, expected {expected}") | |
| out[mod] = feats.numpy().astype(np.float32) | |
| return out | |
| # --------------------------------------------------------------------------- # | |
| # Analysis (CPU, numpy) | |
| # --------------------------------------------------------------------------- # | |
| def _l2n(a: np.ndarray) -> np.ndarray: | |
| return a / (np.linalg.norm(a, axis=-1, keepdims=True) + 1e-8) | |
| def fuse(feats: dict[str, np.ndarray]) -> np.ndarray: | |
| """Per-location fused embedding: mean of the L2-normalised modality tokens.""" | |
| return np.mean([_l2n(f) for f in feats.values()], axis=0) | |
| def pca_rgb(feats: dict[str, np.ndarray], fused: np.ndarray) -> tuple[np.ndarray, dict[str, np.ndarray], np.ndarray]: | |
| """Fit one 3-component PCA basis on every token, so colours are comparable | |
| across modalities and with the fused map. Returns (fused_rgb, per_mod_rgb, explained).""" | |
| pool = np.concatenate([_l2n(f) for f in feats.values()] + [fused], axis=0) | |
| mean = pool.mean(0, keepdims=True) | |
| _, s, vt = np.linalg.svd(pool - mean, full_matrices=False) | |
| basis = vt[:3].T | |
| explained = (s[:3] ** 2) / (s ** 2).sum() | |
| proj_pool = (pool - mean) @ basis | |
| lo = np.percentile(proj_pool, 2, axis=0) | |
| hi = np.percentile(proj_pool, 98, axis=0) | |
| def to_rgb(f): | |
| p = ((f - mean) @ basis - lo) / (hi - lo + 1e-8) | |
| return np.clip(p, 0, 1) | |
| return to_rgb(fused), {m: to_rgb(_l2n(f)) for m, f in feats.items()}, explained | |
| def kmeans(x: np.ndarray, k: int, iters: int = 50, seed: int = 0) -> np.ndarray: | |
| """Cosine k-means (k-means++ init) on L2-normalised features.""" | |
| x = _l2n(x) | |
| rng = np.random.default_rng(seed) | |
| centers = [x[rng.integers(len(x))]] | |
| for _ in range(1, k): | |
| d = 1.0 - np.max(x @ np.stack(centers).T, axis=1) | |
| p = np.clip(d, 0, None) ** 2 | |
| centers.append(x[rng.choice(len(x), p=p / p.sum())] if p.sum() > 0 else x[rng.integers(len(x))]) | |
| c = np.stack(centers) | |
| labels = np.zeros(len(x), dtype=int) | |
| for it in range(iters): | |
| new = np.argmax(x @ c.T, axis=1) | |
| if it and np.array_equal(new, labels): | |
| break | |
| labels = new | |
| for j in range(k): | |
| members = x[labels == j] | |
| if len(members): | |
| c[j] = _l2n(members.mean(0)) | |
| # Relabel by cluster size so colours are stable-ish across runs. | |
| order = np.argsort(-np.bincount(labels, minlength=k)) | |
| remap = np.empty(k, dtype=int) | |
| remap[order] = np.arange(k) | |
| return remap[labels] | |
| def cross_modal_alignment(feats: dict[str, np.ndarray]) -> list[tuple[str, str, float, float]]: | |
| """For each modality pair: mean cosine of same-location tokens vs. different-location tokens.""" | |
| mods = list(feats) | |
| rows = [] | |
| for i in range(len(mods)): | |
| for j in range(i + 1, len(mods)): | |
| a, b = _l2n(feats[mods[i]]), _l2n(feats[mods[j]]) | |
| sim = a @ b.T | |
| n = len(sim) | |
| diag = float(np.trace(sim) / n) | |
| off = float((sim.sum() - np.trace(sim)) / (n * n - n)) | |
| rows.append((mods[i], mods[j], diag, off)) | |
| return rows | |
| # --------------------------------------------------------------------------- # | |
| # Rendering | |
| # --------------------------------------------------------------------------- # | |
| def _cmap(name: str, x01: np.ndarray) -> np.ndarray: | |
| return (colormaps[name](np.clip(x01, 0, 1))[..., :3] * 255).round().astype(np.uint8) | |
| def _stretch(a: np.ndarray) -> np.ndarray: | |
| lo, hi = np.nanpercentile(a, 1), np.nanpercentile(a, 99) | |
| return (a - lo) / (hi - lo + 1e-8) | |
| def render_modality(tile: str, mod: str) -> np.ndarray: | |
| """Render one image modality of a tile as RGB uint8 (H, W, 3).""" | |
| a = _to_chw(G.denormalize_data(load_tile(tile)[mod], mod, MODALITY_INFO)) | |
| if mod == "vis": # [415, 566, 604, 643, 689] nm -> natural-ish RGB | |
| rgb = np.stack([a[3], a[1], a[0]], axis=-1) | |
| return (np.clip(_stretch(rgb), 0, 1) * 255).round().astype(np.uint8) | |
| if mod in ("uv", "nac"): | |
| return _cmap("gray", _stretch(a[0])) | |
| if mod.startswith("dtm"): | |
| return _cmap("terrain", _stretch(a[0])) | |
| if mod.startswith("slope"): | |
| return _cmap("magma", _stretch(a[0])) | |
| if mod.startswith("aspect"): | |
| return _cmap("twilight", (np.degrees(np.arctan2(a[0], a[1])) % 360) / 360) | |
| return _cmap("viridis", _stretch(a[0])) | |
| def upsample(grid_img: np.ndarray, size: int) -> np.ndarray: | |
| """Nearest-neighbour upsample a (g, g[, C]) map to (size, size[, C]).""" | |
| rep = size // grid_img.shape[0] | |
| return np.repeat(np.repeat(grid_img, rep, axis=0), rep, axis=1) | |
| def grid_rgb(flat: np.ndarray, size: int) -> np.ndarray: | |
| g = int(round(np.sqrt(flat.shape[0]))) | |
| img = flat.reshape(g, g, -1) | |
| if img.dtype != np.uint8: | |
| img = (np.clip(img, 0, 1) * 255).round().astype(np.uint8) | |
| return upsample(img, size) | |
| def blend(base: np.ndarray, overlay: np.ndarray, alpha: float) -> np.ndarray: | |
| gray = base.astype(np.float32).mean(-1, keepdims=True).repeat(3, -1) | |
| return ((1 - alpha) * gray + alpha * overlay.astype(np.float32)).round().astype(np.uint8) | |
| def cluster_image(labels: np.ndarray, base: np.ndarray) -> np.ndarray: | |
| k = int(labels.max()) + 1 | |
| cmap = colormaps["tab20" if k > 10 else "tab10"] | |
| colors = (np.array([cmap(i)[:3] for i in range(k)]) * 255).astype(np.uint8) | |
| return blend(base, grid_rgb(colors[labels], base.shape[0]), 0.6) | |
| def similarity_image(fused: np.ndarray, patch: int, base: np.ndarray) -> tuple[np.ndarray, np.ndarray]: | |
| f = _l2n(fused) | |
| sim = f @ f[patch] | |
| lo = np.percentile(sim, 1) | |
| heat = _cmap("inferno", (sim - lo) / (1 - lo + 1e-8)) | |
| img = blend(base, grid_rgb(heat, base.shape[0]), 0.75) | |
| g = int(round(np.sqrt(len(sim)))) | |
| cell = base.shape[0] // g | |
| r, c = divmod(patch, g) | |
| y0, x0 = r * cell, c * cell | |
| img[y0:y0 + cell, [x0, min(x0 + cell - 1, img.shape[1] - 1)]] = (0, 255, 255) | |
| img[[y0, min(y0 + cell - 1, img.shape[0] - 1)], x0:x0 + cell] = (0, 255, 255) | |
| return img, sim | |