lunar-fm-feature-explorer / lfm_features.py
Onyoin's picture
Lunar FM encoder feature explorer
fbe4f04 verified
Raw History Blame Contribute Delete
16.5 kB
"""Encoder-feature plumbing for the NASA-IBM Lunar Foundation Model explorer.
Model / data loading mirrors the authors' reference notebook
(`examples/generate_images.ipynb` in NASA-IMPACT/NASA-IBM-Lunar-Foundation-Model):
same `config.yaml`, modality info, dataset transforms and checkpoint loader.
Only the *encoder* half is run here: raw modalities are patch-embedded
(optionally at a FlexiViT runtime patch size), concatenated with the
register / text tokens, passed through the 12-layer ViT-B encoder, and the
per-patch output tokens are returned for analysis.
Kept free of `gradio` imports so it can be exercised head-less.
"""
from __future__ import annotations
import os
import re
import threading
from functools import lru_cache
from pathlib import Path
import numpy as np
import torch
from huggingface_hub import snapshot_download
from matplotlib import colormaps
from omegaconf import OmegaConf
from tokenizers import Tokenizer
# `ni_lfm` resolves the text tokenizer and metadata binning table relative to
# the repo root, exactly as the reference notebook does with `os.chdir`.
APP_DIR = Path(__file__).resolve().parent
os.chdir(APP_DIR)
from ni_lfm.data.modality_info import setup_modality_transform # noqa: E402
from ni_lfm.models.generate import init_full_input_modality # noqa: E402
from ni_lfm.models.ni_lfm_generation import get_lfm_generation_model # noqa: E402
from ni_lfm.utils import generation as G # noqa: E402
MODEL_REPO = "nasa-ibm-ai4science/NASA-IBM-Lunar-Foundation-Model"
DATA_REPO = "nasa-ibm-ai4science/Sombench-pretraining-data"
TEXT_MODALITIES = ("metadata", "static_maps")
PATCH_SIZES = (16, 8) # 16 = pretraining; 8 = FlexiViT, used for the model card's results
TRACKS: dict[str, dict] = {
"WAC": {
"label": "WAC — regional, 51.2 km tiles",
"folder": "WAC_LowRes",
"parquet": "WAC_LowRes/WAC_LowRes.parquet",
"image_modalities": ["vis", "uv", "dtm", "slope", "aspect"],
"main": "vis",
},
"NAC": {
"label": "NAC — local, 512 m tiles",
"folder": "NAC_HighRes",
"parquet": "NAC_HighRes/NAC_HighRes.parquet",
"image_modalities": ["nac", "dtm_3m", "slope_3m", "aspect_3m"],
"main": "nac",
},
}
MOD_META: dict[str, dict] = {
"vis": {"title": "WAC VIS reflectance", "gsd": "100 m"},
"uv": {"title": "WAC UV reflectance", "gsd": "500 m"},
"dtm": {"title": "Elevation (DTM)", "gsd": "60 m"},
"slope": {"title": "Slope", "gsd": "60 m"},
"aspect": {"title": "Aspect (sin, cos)", "gsd": "60 m"},
"nac": {"title": "NAC reflectance", "gsd": "1 m"},
"dtm_3m": {"title": "Elevation (NAC DTM)", "gsd": "3 m"},
"slope_3m": {"title": "Slope", "gsd": "3 m"},
"aspect_3m": {"title": "Aspect (sin, cos)", "gsd": "3 m"},
"metadata": {"title": "Acquisition geometry (text tokens)", "gsd": "—"},
"static_maps": {"title": "Static map priors (text tokens)", "gsd": "—"},
}
_MODEL_LOCK = threading.Lock()
# --------------------------------------------------------------------------- #
# Startup: weights, config, data
# --------------------------------------------------------------------------- #
def _load_config(path: Path):
"""Load the pretraining config, repairing a duplicated key in the release.
`backbone/config.yaml` ships `text_tokenizer_path: text_tokenizer_path: ...`,
which is not valid YAML, so the line is patched before parsing.
"""
text = path.read_text()
text = re.sub(
r"^(\s*text_tokenizer_path:).*$",
r"\1 ni_lfm/utils/tokenizer/trained/text_tokenizer.json",
text,
count=1,
flags=re.M,
)
return OmegaConf.create(text)
print("Downloading backbone weights ...", flush=True)
WEIGHTS_ROOT = Path(
os.environ.get("LFM_WEIGHTS_DIR")
or snapshot_download(MODEL_REPO, allow_patterns=["backbone/*"])
)
print("Downloading SomBench sample tiles ...", flush=True)
DATA_ROOT = Path(
os.environ.get("LFM_DATA_DIR")
or snapshot_download(DATA_REPO, repo_type="dataset", allow_patterns=["WAC_LowRes/**", "NAC_HighRes/**"])
)
CFG = _load_config(WEIGHTS_ROOT / "backbone" / "config.yaml")
MODALITY_INFO, INPUT_SIZE = G.build_modality_info(CFG)
TEXT_TOKENIZER = Tokenizer.from_file(CFG.data.text_tokenizer_path)
IN_DOMAINS = list(CFG.data.in_domains)
OUT_DOMAINS = list(CFG.data.out_domains)
ALL_DOMAINS = sorted(set(IN_DOMAINS) | set(OUT_DOMAINS))
MODEL_MOD_INFO = G.filter_modality_info(ALL_DOMAINS, MODALITY_INFO)
INPUT_MODALITIES: dict[str, list[str]] = {
track: [*spec["image_modalities"], *TEXT_MODALITIES] for track, spec in TRACKS.items()
}
def _build_datasets() -> dict[str, object]:
datasets = {}
for track, spec in TRACKS.items():
mods = INPUT_MODALITIES[track]
mod_info = G.filter_modality_info(mods, MODALITY_INFO)
datasets[track] = G.LunarGenerationDataset(
data_root=str(DATA_ROOT / spec["folder"]),
index_path=str(DATA_ROOT / spec["parquet"]),
modalities=mods,
modality_transforms=setup_modality_transform(domains=mods, modality_info=mod_info),
center_crop=G.CenterCropImage(target_size=INPUT_SIZE, main_domain=spec["main"]),
max_samples=None,
remove_key_mod_nan=True,
return_file_name=True,
)
return datasets
DATASETS = _build_datasets()
def _build_catalog() -> dict[str, dict]:
"""`"<TRACK>:<idx>"` -> tile description, for every sample tile."""
catalog = {}
for track, ds in DATASETS.items():
for i in range(len(ds)):
r = ds.index.iloc[i]
lat, lon = float(r["CENTER_LATITUDE"]), float(r["CENTER_LONGITUDE"])
name = f"{r['PRODUCT_ID']}_r{int(r['ROW'])}_c{int(r['COL'])}"
catalog[f"{track}:{i}"] = {
"track": track,
"idx": i,
"name": name,
"row": r,
"label": f"{track} · {name} · lat {lat:+.2f}°, lon {lon:+.2f}° · "
f"inc {float(r['INCIDENCE_ANGLE']):.0f}°",
}
return catalog
CATALOG = _build_catalog()
print(f"Sample tiles: {len(CATALOG)} ({', '.join(f'{t}={len(d)}' for t, d in DATASETS.items())})", flush=True)
def tile_choices() -> list[tuple[str, str]]:
return [(t["label"], key) for key, t in CATALOG.items()]
def tile_track(tile: str) -> str:
return CATALOG[tile]["track"]
def tile_info_md(tile: str) -> str:
t = CATALOG[tile]
r = t["row"]
return (
f"**{TRACKS[t['track']]['label']}** · `{t['name']}` · centre "
f"**{float(r['CENTER_LATITUDE']):+.3f}°, {float(r['CENTER_LONGITUDE']):+.3f}°**\n\n"
f"incidence **{float(r['INCIDENCE_ANGLE']):.1f}°** · emission **{float(r['EMISSION_ANGLE']):.1f}°** · "
f"phase **{float(r['PHASE_ANGLE']):.1f}°** · sun azimuth **{float(r['SUB_SOLAR_GROUND_AZIMUTH']):.0f}°**"
)
# --------------------------------------------------------------------------- #
# Model — encoder only, so no VQ tokenizers / diffusion decoders are loaded.
# --------------------------------------------------------------------------- #
print(f"Building model: {CFG.model.name}", flush=True)
MODEL = get_lfm_generation_model(
variant=CFG.model.name,
input_modalities=IN_DOMAINS,
output_modalities=OUT_DOMAINS,
modality_info=MODEL_MOD_INFO,
pretrained_tokenizers=torch.nn.ModuleDict(),
text_tokenizer=TEXT_TOKENIZER,
cfg=CFG,
num_register_tokens=CFG.model.num_register_tokens,
)
G.load_ckpt_for_generation(str(WEIGHTS_ROOT / "backbone" / "checkpoint.pt"), MODEL)
MODEL.eval()
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
MODEL.to(DEVICE)
SAMPLER = MODEL.sampler
ENCDEC = SAMPLER.model
print(f"Model parameters: {sum(p.numel() for p in MODEL.parameters()):,}", flush=True)
@lru_cache(maxsize=32)
def load_tile(tile: str) -> dict:
t = CATALOG[tile]
return G.collate_fn([DATASETS[t["track"]][t["idx"]]])
def _to_chw(x) -> np.ndarray:
a = x.detach().cpu().float().numpy() if torch.is_tensor(x) else np.asarray(x)
return a[0] if a.ndim == 4 else (a[None] if a.ndim == 2 else a)
# --------------------------------------------------------------------------- #
# Encoder forward
# --------------------------------------------------------------------------- #
@torch.no_grad()
def encode(tile: str, modalities: list[str], patch_size: int) -> dict[str, np.ndarray]:
"""Run the encoder on one tile and return per-patch output tokens.
Returns `{image_modality: (grid*grid, 768) float32}`; text modalities are fed
to the encoder (they condition every patch through attention) but have no
spatial layout, so they are not returned.
"""
track = tile_track(tile)
mods = [m for m in modalities if m in INPUT_MODALITIES[track]]
img_mods = [m for m in mods if m not in TEXT_MODALITIES]
if not img_mods:
raise ValueError("Select at least one image modality of this tile's track.")
if patch_size not in PATCH_SIZES:
raise ValueError(f"patch_size must be one of {PATCH_SIZES}")
batch = load_tile(tile)
inputs = G.prepare_model_inputs(
batch=batch, in_domains=mods, modality_info=MODALITY_INFO,
text_tokenizer=TEXT_TOKENIZER, device=DEVICE,
)
mod_dict = {}
for mod, value in inputs.items():
if MODALITY_INFO[mod]["type"] == "img":
h, w = value.shape[-2:]
d = {"tensor": value, "runtime_patch_size": int(patch_size)}
mod_dict[mod] = init_full_input_modality(
d, MODALITY_INFO, mod, DEVICE, eos_id=MODEL.eos_id,
num_tokens=(h // patch_size) * (w // patch_size),
)
else:
mod_dict[mod] = init_full_input_modality(value, MODALITY_INFO, mod, DEVICE, eos_id=MODEL.eos_id)
with _MODEL_LOCK:
enc = {m: ENCDEC.encoder_embeddings[m](d) for m, d in mod_dict.items()}
tokens, emb, mask, mod_mask = SAMPLER.forward_mask_encoder_generation(enc)
x = ENCDEC.forward_encoder(tokens + emb, mask)
x, mod_mask = x[0].float().cpu(), mod_mask[0].cpu()
out = {}
for mod in img_mods:
feats = x[mod_mask == MODALITY_INFO[mod]["id"]]
expected = enc[mod]["x"].shape[1]
if feats.shape[0] != expected:
raise RuntimeError(f"{mod}: got {feats.shape[0]} encoder tokens, expected {expected}")
out[mod] = feats.numpy().astype(np.float32)
return out
# --------------------------------------------------------------------------- #
# Analysis (CPU, numpy)
# --------------------------------------------------------------------------- #
def _l2n(a: np.ndarray) -> np.ndarray:
return a / (np.linalg.norm(a, axis=-1, keepdims=True) + 1e-8)
def fuse(feats: dict[str, np.ndarray]) -> np.ndarray:
"""Per-location fused embedding: mean of the L2-normalised modality tokens."""
return np.mean([_l2n(f) for f in feats.values()], axis=0)
def pca_rgb(feats: dict[str, np.ndarray], fused: np.ndarray) -> tuple[np.ndarray, dict[str, np.ndarray], np.ndarray]:
"""Fit one 3-component PCA basis on every token, so colours are comparable
across modalities and with the fused map. Returns (fused_rgb, per_mod_rgb, explained)."""
pool = np.concatenate([_l2n(f) for f in feats.values()] + [fused], axis=0)
mean = pool.mean(0, keepdims=True)
_, s, vt = np.linalg.svd(pool - mean, full_matrices=False)
basis = vt[:3].T
explained = (s[:3] ** 2) / (s ** 2).sum()
proj_pool = (pool - mean) @ basis
lo = np.percentile(proj_pool, 2, axis=0)
hi = np.percentile(proj_pool, 98, axis=0)
def to_rgb(f):
p = ((f - mean) @ basis - lo) / (hi - lo + 1e-8)
return np.clip(p, 0, 1)
return to_rgb(fused), {m: to_rgb(_l2n(f)) for m, f in feats.items()}, explained
def kmeans(x: np.ndarray, k: int, iters: int = 50, seed: int = 0) -> np.ndarray:
"""Cosine k-means (k-means++ init) on L2-normalised features."""
x = _l2n(x)
rng = np.random.default_rng(seed)
centers = [x[rng.integers(len(x))]]
for _ in range(1, k):
d = 1.0 - np.max(x @ np.stack(centers).T, axis=1)
p = np.clip(d, 0, None) ** 2
centers.append(x[rng.choice(len(x), p=p / p.sum())] if p.sum() > 0 else x[rng.integers(len(x))])
c = np.stack(centers)
labels = np.zeros(len(x), dtype=int)
for it in range(iters):
new = np.argmax(x @ c.T, axis=1)
if it and np.array_equal(new, labels):
break
labels = new
for j in range(k):
members = x[labels == j]
if len(members):
c[j] = _l2n(members.mean(0))
# Relabel by cluster size so colours are stable-ish across runs.
order = np.argsort(-np.bincount(labels, minlength=k))
remap = np.empty(k, dtype=int)
remap[order] = np.arange(k)
return remap[labels]
def cross_modal_alignment(feats: dict[str, np.ndarray]) -> list[tuple[str, str, float, float]]:
"""For each modality pair: mean cosine of same-location tokens vs. different-location tokens."""
mods = list(feats)
rows = []
for i in range(len(mods)):
for j in range(i + 1, len(mods)):
a, b = _l2n(feats[mods[i]]), _l2n(feats[mods[j]])
sim = a @ b.T
n = len(sim)
diag = float(np.trace(sim) / n)
off = float((sim.sum() - np.trace(sim)) / (n * n - n))
rows.append((mods[i], mods[j], diag, off))
return rows
# --------------------------------------------------------------------------- #
# Rendering
# --------------------------------------------------------------------------- #
def _cmap(name: str, x01: np.ndarray) -> np.ndarray:
return (colormaps[name](np.clip(x01, 0, 1))[..., :3] * 255).round().astype(np.uint8)
def _stretch(a: np.ndarray) -> np.ndarray:
lo, hi = np.nanpercentile(a, 1), np.nanpercentile(a, 99)
return (a - lo) / (hi - lo + 1e-8)
def render_modality(tile: str, mod: str) -> np.ndarray:
"""Render one image modality of a tile as RGB uint8 (H, W, 3)."""
a = _to_chw(G.denormalize_data(load_tile(tile)[mod], mod, MODALITY_INFO))
if mod == "vis": # [415, 566, 604, 643, 689] nm -> natural-ish RGB
rgb = np.stack([a[3], a[1], a[0]], axis=-1)
return (np.clip(_stretch(rgb), 0, 1) * 255).round().astype(np.uint8)
if mod in ("uv", "nac"):
return _cmap("gray", _stretch(a[0]))
if mod.startswith("dtm"):
return _cmap("terrain", _stretch(a[0]))
if mod.startswith("slope"):
return _cmap("magma", _stretch(a[0]))
if mod.startswith("aspect"):
return _cmap("twilight", (np.degrees(np.arctan2(a[0], a[1])) % 360) / 360)
return _cmap("viridis", _stretch(a[0]))
def upsample(grid_img: np.ndarray, size: int) -> np.ndarray:
"""Nearest-neighbour upsample a (g, g[, C]) map to (size, size[, C])."""
rep = size // grid_img.shape[0]
return np.repeat(np.repeat(grid_img, rep, axis=0), rep, axis=1)
def grid_rgb(flat: np.ndarray, size: int) -> np.ndarray:
g = int(round(np.sqrt(flat.shape[0])))
img = flat.reshape(g, g, -1)
if img.dtype != np.uint8:
img = (np.clip(img, 0, 1) * 255).round().astype(np.uint8)
return upsample(img, size)
def blend(base: np.ndarray, overlay: np.ndarray, alpha: float) -> np.ndarray:
gray = base.astype(np.float32).mean(-1, keepdims=True).repeat(3, -1)
return ((1 - alpha) * gray + alpha * overlay.astype(np.float32)).round().astype(np.uint8)
def cluster_image(labels: np.ndarray, base: np.ndarray) -> np.ndarray:
k = int(labels.max()) + 1
cmap = colormaps["tab20" if k > 10 else "tab10"]
colors = (np.array([cmap(i)[:3] for i in range(k)]) * 255).astype(np.uint8)
return blend(base, grid_rgb(colors[labels], base.shape[0]), 0.6)
def similarity_image(fused: np.ndarray, patch: int, base: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
f = _l2n(fused)
sim = f @ f[patch]
lo = np.percentile(sim, 1)
heat = _cmap("inferno", (sim - lo) / (1 - lo + 1e-8))
img = blend(base, grid_rgb(heat, base.shape[0]), 0.75)
g = int(round(np.sqrt(len(sim))))
cell = base.shape[0] // g
r, c = divmod(patch, g)
y0, x0 = r * cell, c * cell
img[y0:y0 + cell, [x0, min(x0 + cell - 1, img.shape[1] - 1)]] = (0, 255, 255)
img[[y0, min(y0 + cell - 1, img.shape[0] - 1)], x0:x0 + cell] = (0, 255, 255)
return img, sim