LingBot-Map Tutorial: GPU-Aware Inference and Point Cloud Export LingBot-Map, an open-source streaming 3D reconstruction pipeline, enables GPU-aware inference and point cloud export through a tutorial that auto-tunes frame limits, camera iterations, scale frames, and KV-cache parameters based on detected VRAM. The pipeline, available on GitHub, processes image or video frames to reconstruct scenes and export results as PLY, NPZ, or GLB files, with configurations for different GPU tiers (small <18 GB, medium 18-30 GB, large 30+ GB). In this tutorial, we implement an end-to-end streaming 3D reconstruction pipeline with LingBot-Map https://github.com/robbyant/lingbot-map . We begin by configuring the input source, reconstruction settings, checkpoint selection, and output controls, then probe the available GPU and automatically tune frame limits, camera iterations, scale frames, and KV-cache parameters according to the detected VRAM. We install the repository and its dependencies, download the pretrained checkpoint, preprocess image or video frames, and construct the GCTStream model with streaming attention and long-range trajectory memory. We then perform mixed-precision inference, decode the predicted camera poses and intrinsic parameters, convert depth maps into world-coordinate point clouds, validate the recovered geometry, visualize the reconstructed scene and camera trajectory, and export the results as PLY, NPZ, or GLB artifacts. CFG = { "scene": "courthouse", "image folder": None, "video path": None, "fps": 10, "max frames": None, "stride": None, "checkpoint": "lingbot-map.pt", "image size": 518, "patch size": 14, "use sdpa": True, "mode": "streaming", "num scale frames": None, "keyframe interval": None, "kv cache sliding window": 64, "camera num iterations": None, "offload to cpu": True, "window size": 128, "overlap keyframes": 8, "conf percentile": 55.0, "pixel stride": 2, "max plot points": 60000, "export ply": True, "export glb": False, "launch viser": False, "run ablation": False, "seed": 0, } WORK = "/content" REPO = f"{WORK}/lingbot-map" OUT = f"{WORK}/lingbot out" import os, sys, subprocess, glob, json, time, math, shutil, textwrap os.environ.setdefault "PYTORCH CUDA ALLOC CONF", "expandable segments:True" os.makedirs OUT, exist ok=True def sh cmd, check=True : p = subprocess.run cmd, shell=True, capture output=True, text=True if p.returncode = 0 and check: print p.stdout -3000: ; print p.stderr -3000: raise RuntimeError f"command failed: {cmd}" return p def probe gpu : try: q = sh "nvidia-smi --query-gpu=name,memory.total --format=csv,noheader", check=False if q.returncode = 0 or not q.stdout.strip : return None, 0.0 name, mem = x.strip for x in q.stdout.strip .split "\n" 0 .split "," return name, float mem.split 0 / 1024.0 except Exception: return None, 0.0 GPU NAME, VRAM GB = probe gpu print "=" 78 print f"GPU : {GPU NAME or 'NONE — enable Runtime Change runtime type GPU'}" print f"VRAM : {VRAM GB:.1f} GB" try: import psutil print f"System RAM : {psutil.virtual memory .total/2 30:.1f} GB" except Exception: pass print f"Free disk : {shutil.disk usage WORK .free/2 30:.1f} GB checkpoint is 4.6 GB " print "=" 78 if GPU NAME is None: raise SystemExit "This model needs a CUDA GPU. Switch the Colab runtime to GPU." if VRAM GB < 18: AUTO = dict max frames=48, num scale frames=4, camera num iterations=2, kvsw=48 tier = "small <18 GB " elif VRAM GB < 30: AUTO = dict max frames=96, num scale frames=8, camera num iterations=4, kvsw=64 tier = "medium 18-30 GB " else: AUTO = dict max frames=240, num scale frames=8, camera num iterations=4, kvsw=64 tier = "large 30+ GB " if CFG "max frames" is None: CFG "max frames" = AUTO "max frames" if CFG "num scale frames" is None: CFG "num scale frames" = AUTO "num scale frames" if CFG "camera num iterations" is None: CFG "camera num iterations" = AUTO "camera num iterations" if VRAM GB < 18: CFG "kv cache sliding window" = AUTO "kvsw" print f"Auto-tuned for {tier}: max frames={CFG 'max frames' }, " f"scale frames={CFG 'num scale frames' }, " f"cam iters={CFG 'camera num iterations' }, " f"kv window={CFG 'kv cache sliding window' }" print "Raise CFG 'max frames' if you have headroom — reconstruction quality " "improves a lot with more views.\n" We define the configuration parameters that control the input source, model behavior, inference mode, point-cloud generation, and export settings. We inspect the available GPU, system memory, and disk space before automatically adjusting frame limits, scale frames, camera iterations, and KV-cache size according to the detected VRAM. We also create reusable shell and GPU-probing functions that prepare the Colab environment for the remaining reconstruction workflow. if not os.path.isdir REPO : print "Cloning repo shallow ..." sh f"git clone --depth 1 https://github.com/Robbyant/lingbot-map.git {REPO}" print "Installing deps ..." sh "pip install -q einops safetensors huggingface hub" sh f"pip install -q -e {REPO} --no-deps", check=False if REPO not in sys.path: sys.path.insert 0, REPO import importlib importlib.invalidate caches import cv2, numpy as np print f"numpy {np. version } | opencv {cv2. version }" from huggingface hub import hf hub download print f"\nFetching {CFG 'checkpoint' } from robbyant/lingbot-map ..." t0 = time.time CKPT = hf hub download repo id="robbyant/lingbot-map", filename=CFG "checkpoint" , cache dir=f"{WORK}/hf cache" print f" - {CKPT} {os.path.getsize CKPT /2 30:.2f} GB, {time.time -t0:.0f}s " We clone the LingBot-Map repository and install the required dependencies without replacing Colab’s existing NumPy and OpenCV packages. We register the repository in the Python environment and verify the installed library versions to ensure that the runtime is ready for execution. We then download the selected pretrained LingBot-Map checkpoint from Hugging Face and store its local path for model initialization. python import torch import matplotlib.pyplot as plt from lingbot map.models.gct stream import GCTStream from lingbot map.utils.load fn import load and preprocess images from lingbot map.utils.pose enc import pose encoding to extri intri from lingbot map.utils.geometry import closed form inverse se3, closed form inverse se3 general, depth to world coords points, torch.manual seed CFG "seed" ; np.random.seed CFG "seed" DEVICE = torch.device "cuda" CC = torch.cuda.get device capability DTYPE = torch.bfloat16 if CC 0 = 8 else torch.float16 print f"torch {torch. version } | sm {CC 0 }{CC 1 } | inference dtype {DTYPE}" def extract video frames video path, out dir, fps=10 : os.makedirs out dir, exist ok=True cap = cv2.VideoCapture video path src fps = cap.get cv2.CAP PROP FPS or 30 interval = max 1, round src fps / fps paths, idx = , 0 while True: ok, frame = cap.read if not ok: break if idx % interval == 0: p = os.path.join out dir, f"{len paths :06d}.jpg" cv2.imwrite p, frame paths.append p idx += 1 cap.release print f" decoded {idx} frames @ {src fps:.1f} fps - kept {len paths } every {interval} " return paths def gather paths : if CFG "video path" : src = f"{OUT}/video frames" return extract video frames CFG "video path" , src, CFG "fps" , src folder = CFG "image folder" or f"{REPO}/example/{CFG 'scene' }" if not os.path.isdir folder : raise FileNotFoundError folder paths = sorted sum glob.glob os.path.join folder, f" {e}" for e in ".jpg", ".jpeg", ".png", ".JPG", ".PNG" , if not paths: raise FileNotFoundError f"no images in {folder}" return paths, folder print "\n 5 Loading frames" all paths, SRC FOLDER = gather paths print f" source: {SRC FOLDER} {len all paths } frames available " stride = CFG "stride" if stride is None: stride = max 1, math.ceil len all paths / CFG "max frames" paths = all paths ::stride :CFG "max frames" print f" stride={stride} - using {len paths } frames " f" covering { len paths -1 stride+1}/{len all paths } of the sequence " t0 = time.time images = load and preprocess images paths, mode="crop", image size=CFG "image size" , patch size=CFG "patch size" , S, , H, W = images.shape n tok = H // CFG "patch size" W // CFG "patch size" print f" tensor {tuple images.shape } in 0,1 | {W}x{H} | ~{n tok} patch tokens/frame" f" | {time.time -t0:.1f}s" k = min 6, S fig, ax = plt.subplots 1, k, figsize= 3 k, 2.4 for i, a in enumerate np.atleast 1d ax : a.imshow images int i S - 1 / max k - 1, 1 .permute 1, 2, 0 .numpy a.set title f"frame {int i S-1 /max k-1,1 }", fontsize=9 ; a.axis "off" plt.suptitle "Input frames after crop/resize " ; plt.tight layout ; plt.show We import PyTorch, LingBot-Map components, geometry utilities, and visualization libraries before selecting the appropriate inference precision for the available GPU. We load frames from a bundled scene, custom image directory, or video file and uniformly sample them according to the configured frame limit and stride. We preprocess the selected images into model-ready tensors and preview representative frames after cropping and resizing. print "\n 6 Building GCTStream" model = GCTStream img size=CFG "image size" , patch size=CFG "patch size" , enable 3d rope=True, max frame num=1024, kv cache sliding window=CFG "kv cache sliding window" , kv cache scale frames=CFG "num scale frames" , kv cache cross frame special=True, kv cache include scale frames=True, use sdpa=CFG "use sdpa" , camera num iterations=CFG "camera num iterations" , n par = sum p.numel for p in model.parameters print f" {n par/1e9:.2f} B params | heads: " f"camera={model.camera head is not None}, depth={model.depth head is not None}, " f"point={model.point head is not None}" print " loading state dict ..." t0 = time.time try: ckpt = torch.load CKPT, map location="cpu", weights only=False, mmap=True except Exception: ckpt = torch.load CKPT, map location="cpu", weights only=False sd = ckpt.get "model", ckpt missing, unexpected = model.load state dict sd, strict=False del ckpt, sd print f" loaded in {time.time -t0:.0f}s | missing={len missing } unexpected={len unexpected }" if len missing 50: print " lots of missing keys — wrong checkpoint for this model class?" model = model.to DEVICE .eval if model.aggregator is not None: model.aggregator = model.aggregator.to dtype=DTYPE torch.cuda.empty cache print f" GPU after load: {torch.cuda.memory allocated /2 30:.2f} GB allocated, " f"{torch.cuda.memory reserved /2 30:.2f} GB reserved" kfi = CFG "keyframe interval" if kfi is None: kfi = math.ceil S / 320 if CFG "mode" == "streaming" and S 320 else 1 print f"\n 7 {CFG 'mode' } inference | frames={S} | keyframe interval={kfi} | dtype={DTYPE}" torch.cuda.reset peak memory stats out dev = torch.device "cpu" if CFG "offload to cpu" else None t0 = time.time with torch.no grad , torch.amp.autocast "cuda", dtype=DTYPE : if CFG "mode" == "streaming": preds = model.inference streaming images, num scale frames=CFG "num scale frames" , keyframe interval=kfi, output device=out dev, else: preds = model.inference windowed images, window size=CFG "window size" , overlap size=16, overlap keyframes=CFG "overlap keyframes" , num scale frames=CFG "num scale frames" , keyframe interval=kfi, output device=out dev, elapsed = time.time - t0 print f" done in {elapsed:.1f}s - {S/elapsed:.2f} FPS" print f" GPU peak: {torch.cuda.max memory allocated /2 30:.2f} GB allocated " f"/ {torch.cuda.max memory reserved /2 30:.2f} GB reserved" print f" outputs: " + ", ".join f"{k}{tuple v.shape }" for k, v in preds.items if torch.is tensor v try: print f" kv cache: {model.get kv cache info }" except Exception: pass model.clean kv cache torch.cuda.empty cache We construct the GCTStream model with streaming KV-cache controls, camera refinement, 3D rotary embeddings, and scaled dot-product attention. We load the pretrained checkpoint, move the model to the GPU, cast the aggregation trunk to the selected precision, and report its parameter count and memory usage. We then run streaming or windowed inference, record processing speed and peak GPU consumption, and collect the predicted images, depths, confidence maps, and camera pose encodings. python print "\n 8 Decoding camera poses" def decode poses pose enc, hw : extr, intr = pose encoding to extri intri pose enc, hw E = torch.zeros extr.shape :-2 , 4, 4 , device=extr.device, dtype=extr.dtype E ..., :3, :4 = extr E ..., 3, 3 = 1.0 E = closed form inverse se3 general E return E ..., :3, :4 , intr def unbatch t : return t 0 if torch.is tensor t and t.shape 0 == 1 else t extrinsic, intrinsic = decode poses preds "pose enc" .float , H, W extrinsic = unbatch extrinsic .cpu .numpy intrinsic = unbatch intrinsic .cpu .numpy depth = unbatch preds "depth" .float .cpu .numpy depth conf = unbatch preds "depth conf" .float .cpu .numpy rgb = unbatch preds "images" .float .cpu .numpy c2w = closed form inverse se3 extrinsic cam centers = c2w :, :3, 3 path len = float np.linalg.norm np.diff cam centers, axis=0 , axis=1 .sum print f" focal px : {intrinsic 0,0,0 :.1f} principal pt " f" {intrinsic 0,0,2 :.1f}, {intrinsic 0,1,2 :.1f} " print f" depth range : {np.percentile depth,1 :.3f} .. {np.percentile depth,99 :.3f} " f" arbitrary but metric-consistent units " print f" trajectory len : {path len:.3f} | extent " f"{np.ptp cam centers,axis=0 .round 3 .tolist }" print "\n 9 Unprojecting depth to a world point cloud" def conf threshold conf, pct : sub = conf ::max 1, len conf // 20 .ravel return float np.percentile sub, pct THR = conf threshold depth conf, CFG "conf percentile" print f" conf threshold @ {CFG 'conf percentile' }th pct = {THR:.3f}" ps = CFG "pixel stride" pts all, col all, frame id = , , for i in range S : wp, , valid = depth to world coords points depth i .squeeze -1 , extrinsic i , intrinsic i m = valid & depth conf i THR m sub = np.zeros like m ; m sub ::ps, ::ps = True m = m & m sub if not m.any : continue pts all.append wp m .astype np.float32 col all.append rgb i .transpose 1, 2, 0 m .astype np.float32 frame id.append np.full int m.sum , i, dtype=np.int32 points = np.concatenate pts all, 0 colors = np.clip np.concatenate col all, 0 , 0, 1 frames of = np.concatenate frame id, 0 del pts all, col all, frame id print f" {len points :,} points kept " f" {100 len points / S H W :.1f}% of all pixels, stride {ps}, conf {THR:.2f} " i = S // 2 wp, , valid = depth to world coords points depth i .squeeze -1 , extrinsic i , intrinsic i m = valid & depth conf i THR ratio = np.linalg.norm wp m - cam centers i , axis=1 / np.maximum depth i .squeeze -1 m , 1e-6 print f" sanity: median ||p - cam|| / depth = {np.median ratio :.3f} " f" expected ~1.00-1.15 - {'OK' if 0.98 < np.median ratio < 1.35 else 'SUSPECT'}" lo, hi = np.percentile points, 1, 99 , axis=0 scene scale = float np.linalg.norm hi - lo print f" scene diagonal : {scene scale:.3f}" We decode the predicted pose encodings into camera extrinsic and intrinsic matrices and calculate the reconstructed camera trajectory. We convert every confidence-filtered depth map into world-coordinate points while preserving the corresponding RGB values and source-frame identifiers. We also perform a geometric quality check and calculate robust scene boundaries to verify that the reconstructed point cloud follows the expected camera and depth conventions. print "\n 10 Plots" k = min 4, S idxs = np.linspace 0, S - 1, k .astype int fig, axes = plt.subplots 3, k, figsize= 3.1 k, 7.2 axes = np.atleast 2d axes for c, i in enumerate idxs : axes 0, c .imshow rgb i .transpose 1, 2, 0 ; axes 0, c .set title f"frame {i}", fontsize=9 d = depth i .squeeze -1 axes 1, c .imshow d, cmap="turbo", vmin=np.percentile d, 2 , vmax=np.percentile d, 98 axes 2, c .imshow depth conf i THR, cmap="gray" for r, lbl in enumerate "RGB", "depth", f"conf {THR:.2f}" : axes r, 0 .set ylabel lbl, fontsize=10 for a in axes.ravel : a.set xticks ; a.set yticks plt.tight layout ; plt.savefig f"{OUT}/depth strip.png", dpi=110 ; plt.show plt.figure figsize= 11, 3 plt.subplot 1, 2, 1 plt.hist depth conf ::max 1, S // 15 .ravel , bins=120, color=" 4477aa" plt.axvline THR, color="crimson", ls="--", label=f"threshold {THR:.2f}" plt.yscale "log" ; plt.xlabel "depth confidence" ; plt.ylabel "pixels log " plt.legend ; plt.title "Confidence distribution" plt.subplot 1, 2, 2 plt.plot depth conf i THR .mean for i in range S , lw=1.4, color=" 228833" plt.xlabel "frame" ; plt.ylabel "fraction kept" ; plt.ylim 0, 1 plt.title "Per-frame confident-pixel ratio" plt.tight layout ; plt.show fig = plt.figure figsize= 11, 4.2 axA = fig.add subplot 1, 2, 1, projection="3d" axA.plot cam centers.T, color=" cc3311", lw=1.8 axA.scatter cam centers 0 , s=45, c="green", label="start" axA.scatter cam centers -1 , s=45, c="black", label="end" sub = points np.random.choice len points , min 4000, len points , replace=False axA.scatter sub.T, s=0.4, c=" bbbbbb", alpha=0.35 axA.set title "Camera trajectory 3D " ; axA.legend fontsize=8 axB = fig.add subplot 1, 2, 2 axB.plot cam centers :, 0 , cam centers :, 2 , color=" cc3311", lw=1.8 axB.scatter sub :, 0 , sub :, 2 , s=0.4, c=" bbbbbb", alpha=0.35 axB.set xlabel "x" ; axB.set ylabel "z" ; axB.set aspect "equal" axB.set title "Top-down x-z " plt.tight layout ; plt.savefig f"{OUT}/trajectory.png", dpi=110 ; plt.show import plotly.graph objects as go n show = min CFG "max plot points" , len points sel = np.random.choice len points , n show, replace=False P, C = points sel , colors sel 255 .astype np.uint8 inb = np.all P = lo & P <= hi , axis=1 P, C = P inb , C inb fig = go.Figure go.Scatter3d x=P :, 0 , y=P :, 1 , z=P :, 2 , mode="markers", marker=dict size=1.2, color= f"rgb {r},{g},{b} " for r, g, b in C , name="points", hoverinfo="skip" , go.Scatter3d x=cam centers :, 0 , y=cam centers :, 1 , z=cam centers :, 2 , mode="lines+markers", line=dict color="red", width=4 , marker=dict size=2, color="red" , name="camera path" , fig.update layout height=680, margin=dict l=0, r=0, t=28, b=0 , title=f"LingBot-Map reconstruction — {len P :,} of {len points :,} points", scene=dict aspectmode="data", xaxis=dict visible=False , yaxis=dict visible=False , zaxis=dict visible=False , bgcolor="rgb 15,15,20 " fig.show def write ply path, xyz, rgb01 : rgb8 = np.clip rgb01, 0, 1 255 .astype np.uint8 hdr = f"ply\nformat binary little endian 1.0\nelement vertex {len xyz }\n" "property float x\nproperty float y\nproperty float z\n" "property uchar red\nproperty uchar green\nproperty uchar blue\n" "end header\n" dt = np.dtype "x", "