Download serve_clef.py from dbirks/clef-omni-nvfp4: direct link, hf CLI and curl.
- Browser
- Download file 40 kB
-
https://huggingface.co/dbirks/clef-omni-nvfp4/resolve/main/serve_clef.py
- Command line
-
hf download hf://dbirks/clef-omni-nvfp4/serve_clef.py
-
curl -L -o serve_clef.py https://huggingface.co/dbirks/clef-omni-nvfp4/resolve/main/serve_clef.py
40 kB
| # /// script | |
| # requires-python = ">=3.11" | |
| # dependencies = [ | |
| # "torch>=2.11", | |
| # "torchvision>=0.26", | |
| # "transformers==5.10.2", | |
| # "accelerate>=1.0", | |
| # "safetensors>=0.4", | |
| # "huggingface_hub>=1.0", | |
| # "numpy", | |
| # "pillow", | |
| # "av", | |
| # "fastapi", | |
| # "uvicorn[standard]", | |
| # "auto-round==0.16.0", | |
| # ] | |
| # /// | |
| """Serve Clef-Omni (Cloudflare) or one of its quantizations as a SystemOne decision API. | |
| uv run serve_clef.py --model dbirks/clef-omni-nvfp4 # HTTP server on :8000 | |
| uv run serve_clef.py --model dbirks/clef-omni-int4 --demo # one demo decision, then exit | |
| uv run serve_clef.py --model Cloudflare/clef-omni --request req.json | |
| Endpoints: POST /v1/systemone ({"model", "state", "questions", optional "images"/"audio"/"videos"}), | |
| GET /healthz (200 once the model is loaded). Answers come from Cloudflare's own `systemone()` in the | |
| repo's joint_schema_model.py: one forward pass, per-option probabilities from the BF16 joint head. | |
| Checkpoint kinds, detected from the repo's config: | |
| * BF16 original (Cloudflare/clef-omni): Cloudflare's load_release_model, unchanged. | |
| * AutoRound INT4 (quant_method auto-round): transformers loads it with real int4 kernels. | |
| * ModelOpt NVFP4 / NVFP4A16 (hf_quant_config.json) and AutoRound NVFP4 / NVFP4A16 (auto-round, | |
| data_type nv_fp): transformers cannot load packed NVFP4 (AutoRound has no backend for its own | |
| weight-only nv_fp), so the FP4 experts stay PACKED in VRAM (~22 GB for Clef) and each MoE layer is | |
| dequantized just in time to BF16 (bit-exact quantized values) for transformers' grouped_mm, then freed; | |
| --no-packed-nvfp4 dequantizes everything on load instead (~60 GB, BF16 speed). For W4A4 checkpoints, | |
| --simulate-fp4-activations also rounds the expert inputs to NVFP4 exactly as the quantizer calibrated | |
| them (per-16 block scales in FP8 E4M3 under the exported per-tensor input scale), reproducing the | |
| quantized model's answers; it is slow, so it is off by default (and it uses the up-front dequant). | |
| Native FP4 execution needs an engine such as vLLM, plus a hidden-state path into the head. | |
| * FP8 block (quant_method fp8, weight_block_size [128, 128], Qwen/DeepSeek layout from | |
| quant_fp8_block.py): experts stay FP8 + fp32 per-tile scales in VRAM (~32 GiB for Clef) and are | |
| dequantized per layer like NVFP4 (W8A16); --no-packed-fp8 dequantizes on load instead. | |
| --simulate-fp8-activations adds the dynamic 1x128 per-token-group FP8 rounding of expert inputs (W8A8). | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import sys | |
| import threading | |
| import time | |
| from pathlib import Path | |
| import torch | |
| E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0]) | |
| E2M1_PAIRS = torch.stack((E2M1[torch.arange(256) & 15], E2M1[torch.arange(256) >> 4]), dim=-1) # byte -> (lo, hi) | |
| def log(*a): | |
| print(f"[{time.strftime('%FT%T')}]", *a, file=sys.stderr, flush=True) | |
| def resolve(model, revision): | |
| p = Path(model) | |
| if p.is_dir(): | |
| return p | |
| from huggingface_hub import snapshot_download | |
| return Path(snapshot_download(model, revision=revision)) | |
| def kind_of(path): | |
| if (path / "hf_quant_config.json").exists(): | |
| algo = json.loads((path / "hf_quant_config.json").read_text())["quantization"]["quant_algo"] | |
| return f"modelopt:{algo}" | |
| qc = json.loads((path / "config.json").read_text()).get("quantization_config") or {} | |
| method = str(qc.get("quant_method", "")).replace("_", "-") | |
| if method == "fp8": | |
| block = qc.get("weight_block_size") | |
| if not block or len(block) != 2 or qc.get("fmt", "e4m3") != "e4m3": | |
| raise SystemExit(f"unsupported fp8 checkpoint (want e4m3 with weight_block_size): {qc}") | |
| return "fp8-block" | |
| if method == "auto-round": | |
| if "nv_fp" in str(qc.get("data_type", "")): | |
| if qc.get("bits") != 4 or qc.get("group_size") != 16: | |
| raise SystemExit(f"unsupported AutoRound nv_fp scheme: bits={qc.get('bits')} group_size={qc.get('group_size')}") | |
| return "auto-round:nvfp4" if (qc.get("act_bits") or 16) <= 4 else "auto-round:nvfp4a16" | |
| return f"auto-round:{qc.get('bits', '?')}bit" | |
| if method: | |
| raise SystemExit(f"unsupported quantization_config.quant_method={method!r}") | |
| return "bf16" | |
| def unpack_e2m1(packed): | |
| """Two E2M1 codes per byte, low nibble first, bit 3 = sign (ModelOpt and AutoRound/llm-compressor alike).""" | |
| return E2M1_PAIRS.to(packed.device)[packed.int()].reshape(*packed.shape[:-1], -1) # one gather per byte | |
| def recip(t): | |
| return torch.where(t == 0, torch.zeros_like(t), 1.0 / t) | |
| def dequant_nvfp4(packed, scale, scale_2, block=16): | |
| """ModelOpt NVFP4: e4m3 scale per 16, fp32 global scale is a multiplier.""" | |
| vals = unpack_e2m1(packed) | |
| vals = vals.view(*vals.shape[:-1], -1, block) * (scale.float() * scale_2.float()).unsqueeze(-1) | |
| return vals.reshape(*packed.shape[:-1], -1).to(torch.bfloat16) | |
| def dequant_nvfp4_ar(packed, scale, global_scale, block=16): | |
| """AutoRound NVFP4 (llm-compressor packing): the fp32 weight_global_scale is a DIVISOR. Same op order as | |
| AutoRound's own NVFP4QuantLinear._dequant_nvfp4_tensor: (fp4 * 1/global_scale) * e4m3 scale, in fp32.""" | |
| if scale.dtype != torch.float8_e4m3fn: | |
| raise RuntimeError(f"expected FP8 E4M3 weight_scale, got {scale.dtype}") | |
| vals = unpack_e2m1(packed) * recip(global_scale.float()) | |
| vals = vals.view(*vals.shape[:-1], -1, block) * scale.float().unsqueeze(-1) | |
| return vals.reshape(*packed.shape[:-1], -1).to(torch.bfloat16) | |
| def dequant_fp8_block(w, scale_inv): | |
| """FP8 block checkpoints: e4m3 times the fp32 per-tile weight_scale_inv, tile size from the scale grid. | |
| Same math as transformers' Fp8Dequantize._dequantize_one (fp32 product, then BF16). Leading dims allowed.""" | |
| *lead, rows, cols = w.shape | |
| sr, sc = scale_inv.shape[-2:] | |
| if rows % sr or cols % sc: | |
| raise RuntimeError(f"fp8 weight {tuple(w.shape)} does not tile by scale grid {tuple(scale_inv.shape)}") | |
| q = w.float().reshape(*lead, sr, rows // sr, sc, cols // sc) | |
| return (q * scale_inv.float()[..., :, None, :, None]).reshape(w.shape).to(torch.bfloat16) | |
| def fp8_act_quant(x, group=128): | |
| """Dynamic per-token-group (1 x 128) FP8 E4M3 activation rounding, as the W8A8 block-FP8 kernels do | |
| (scale = group amax / 448, vLLM per_token_group_quant_fp8 / DeepSeek act_quant); returns BF16.""" | |
| shape, dtype = x.shape, x.dtype | |
| xf = x.float().reshape(-1, shape[-1] // group, group) | |
| scale = xf.abs().amax(-1, keepdim=True).clamp(min=1e-10) / 448.0 | |
| return ((xf / scale).clamp(-448.0, 448.0).to(torch.float8_e4m3fn).float() * scale).reshape(shape).to(dtype) | |
| def round_fp8_expert_inputs(experts): | |
| """Round both expert GEMM inputs (permuted hidden states, gated intermediate) to FP8 per 1x128 token group. | |
| Dynamic scales are per row, so rounding before the permutation equals rounding after it; works on the | |
| packed modules and on transformers' grouped_mm experts (both call self._apply_gate).""" | |
| inner, gate = experts.forward, experts._apply_gate | |
| experts._apply_gate = lambda gate_up: fp8_act_quant(gate(gate_up)) | |
| experts.forward = lambda h, top_k_index, top_k_weights: inner(fp8_act_quant(h), top_k_index, top_k_weights) | |
| def round_e2m1(a): | |
| """|x| to the nearest E2M1 magnitude, ties to even (same as AutoRound's cast_to_fp4).""" | |
| return torch.where(a <= 0.25, 0.0, torch.where(a < 0.75, 0.5, torch.where(a <= 1.25, 1.0, torch.where( | |
| a < 1.75, 1.5, torch.where(a <= 2.5, 2.0, torch.where(a < 3.5, 3.0, torch.where(a <= 5.0, 4.0, 6.0))))))) | |
| def fp4_fake_quant(x, global_scale, block=16): | |
| """ModelOpt's dynamic NVFP4 activation rounding (kernels/quantization/gemm/fp4_kernel_hopper.py).""" | |
| shape, dtype = x.shape, x.dtype | |
| xf = x.float().reshape(-1, shape[-1] // block, block) | |
| amax = xf.abs().amax(-1, keepdim=True) | |
| scale = (amax / (6.0 * global_scale)).clamp(max=448.0).to(torch.float8_e4m3fn).float() * global_scale | |
| scale = torch.where(scale >= 1e-5, scale, torch.ones_like(scale)) | |
| return (torch.sign(xf) * round_e2m1(xf.abs() / scale) * scale).reshape(shape).to(dtype) | |
| def fp4_fake_quant_ar(x, global_scale, block=16): | |
| """AutoRound's static-global-scale NVFP4 activation rounding (data_type/nvfp.py ref_nvfp4_quant), op for op. | |
| global_scale is the exported input_global_scale = 448*6/amax, the reciprocal of ModelOpt's input_scale.""" | |
| shape, dtype = x.shape, x.dtype | |
| xf = x.float().reshape(-1, block) | |
| scale = (global_scale * (xf.abs().amax(-1, keepdim=True) * (1.0 / 6.0))).clamp(-448.0, 448.0) | |
| inv = recip(scale.to(torch.float8_e4m3fn).float() * recip(global_scale)) | |
| xs = (xf * inv).clamp(-6.0, 6.0) | |
| return (torch.sign(xs) * round_e2m1(xs.abs()) * recip(inv)).reshape(shape).to(dtype) | |
| def simulate_fp4_activations(experts, gu_quant, dn_quant): | |
| """Round expert inputs to NVFP4 inside a fused-experts module (eager path calls F.linear per expert). | |
| gu_quant / dn_quant: one rounding function per expert (AutoRound calibrates each expert separately).""" | |
| import torch.nn.functional as F | |
| linear, inner = F.linear, experts.forward | |
| gu, dn = experts.gate_up_proj, experts.down_proj | |
| def owner(w): | |
| p = w.data_ptr() | |
| for param, qs in ((gu, gu_quant), (dn, dn_quant)): | |
| start, step = param.data_ptr(), param[0].numel() * param.element_size() | |
| if start <= p < start + len(qs) * step: | |
| return qs[(p - start) // step] | |
| return None | |
| def q_linear(x, w, b=None): | |
| q = owner(w) | |
| return linear(q(x) if q is not None else x, w, b) | |
| def forward(*a, **k): | |
| F.linear = q_linear | |
| try: | |
| return inner(*a, **k) | |
| finally: | |
| F.linear = linear | |
| experts.forward = forward | |
| class PackedNVFP4Experts(torch.nn.Module): | |
| """All experts of one MoE layer kept as packed NVFP4 in VRAM: gate|up and down as [E, out, in/2] uint8 | |
| codes, [E, out, in/16] E4M3 block scales and a per-row fp32 global scale (gate and up rows each carry | |
| their own tensor's scale). forward dequantizes the layer into transient BF16 [E, out, in] weights (the | |
| transformers-5 parameter layout, bit-identical to the up-front dequant) and runs transformers' | |
| grouped_mm experts forward with is_transposed=False; the temporaries are freed on return.""" | |
| def __init__(self, experts, divisor, chunk_bytes=256 << 20): | |
| super().__init__() | |
| self.num_experts, self.act_fn, self.divisor = experts.num_experts, experts.act_fn, divisor | |
| self.shapes = {"gate_up": (2 * experts.intermediate_dim, experts.hidden_dim), | |
| "down": (experts.hidden_dim, experts.intermediate_dim)} | |
| self.chunk_bytes = chunk_bytes # bounds the dequant transient (int32 index + fp32 values: ~32 B per packed byte) | |
| def set_weights(self, name, parts): | |
| """parts: per expert, the (packed, e4m3 scale, global scale) of each projection, concatenated by row.""" | |
| rows, cols = self.shapes[name] | |
| packed = torch.stack([torch.cat([p for p, _, _ in ps]) for ps in parts]) | |
| scale = torch.stack([torch.cat([s for _, s, _ in ps]) for ps in parts]) | |
| gs = torch.stack([torch.cat([g.float().reshape(1, 1).expand(p.shape[0], 1) for p, _, g in ps]) for ps in parts]) | |
| if (tuple(packed.shape) != (self.num_experts, rows, cols // 2) or packed.dtype != torch.uint8 | |
| or tuple(scale.shape) != (self.num_experts, rows, cols // 16) or scale.dtype != torch.float8_e4m3fn): | |
| raise RuntimeError(f"unexpected packed NVFP4 {name}: {tuple(packed.shape)} {packed.dtype}, " | |
| f"scales {tuple(scale.shape)} {scale.dtype}; want E={self.num_experts} [{rows}, {cols}]") | |
| self.register_buffer(f"{name}_packed", packed) | |
| self.register_buffer(f"{name}_scale", scale) | |
| self.register_buffer(f"{name}_global", gs.contiguous()) | |
| def weight(self, name): | |
| packed, scale, gs = (getattr(self, f"{name}_{t}") for t in ("packed", "scale", "global")) | |
| out = torch.empty(*packed.shape[:-1], packed.shape[-1] * 2, device=packed.device, dtype=torch.bfloat16) | |
| deq = dequant_nvfp4_ar if self.divisor else dequant_nvfp4 | |
| step = max(1, self.chunk_bytes // (32 * packed[0].numel())) | |
| for e in range(0, self.num_experts, step): | |
| out[e:e + step] = deq(packed[e:e + step], scale[e:e + step], gs[e:e + step]) | |
| return out | |
| def _apply_gate(self, gate_up): | |
| gate, up = gate_up.chunk(2, dim=-1) | |
| return self.act_fn(gate) * up | |
| def forward(self, hidden_states, top_k_index, top_k_weights): | |
| from types import SimpleNamespace | |
| from transformers.integrations.moe import grouped_mm_experts_forward | |
| view = SimpleNamespace(num_experts=self.num_experts, has_gate=True, has_bias=False, is_transposed=False, | |
| gate_up_proj=self.weight("gate_up"), down_proj=self.weight("down"), | |
| _apply_gate=self._apply_gate) | |
| return grouped_mm_experts_forward(view, hidden_states, top_k_index, top_k_weights) | |
| class PackedFP8Experts(torch.nn.Module): | |
| """All experts of one MoE layer kept as FP8 E4M3 in VRAM: gate|up and down as [E, out, in] float8 plus | |
| [E, out/128, in/128] fp32 weight_scale_inv (gate and up tiles concatenated by row). forward dequantizes | |
| the layer into transient BF16 weights (bit-identical to the up-front dequant) and runs transformers' | |
| grouped_mm experts forward, like PackedNVFP4Experts.""" | |
| def __init__(self, experts, chunk_bytes=256 << 20): | |
| super().__init__() | |
| self.num_experts, self.act_fn = experts.num_experts, experts.act_fn | |
| self.shapes = {"gate_up": (2 * experts.intermediate_dim, experts.hidden_dim), | |
| "down": (experts.hidden_dim, experts.intermediate_dim)} | |
| self.chunk_bytes = chunk_bytes # bounds the dequant transient (fp32 values: 4 B per weight) | |
| def set_weights(self, name, parts): | |
| """parts: per expert, the (fp8 weight, fp32 scale_inv) of each projection, concatenated by row.""" | |
| rows, cols = self.shapes[name] | |
| w = torch.stack([torch.cat([q for q, _ in ps]) for ps in parts]) | |
| scale = torch.stack([torch.cat([s for _, s in ps]) for ps in parts]).float() | |
| if (tuple(w.shape) != (self.num_experts, rows, cols) or w.dtype != torch.float8_e4m3fn | |
| or rows % scale.shape[1] or cols % scale.shape[2] | |
| or any(p[0].shape[0] * scale.shape[1] != rows * p[1].shape[0] for p in parts[0])): | |
| raise RuntimeError(f"unexpected FP8 {name}: {tuple(w.shape)} {w.dtype}, scales {tuple(scale.shape)}; " | |
| f"want E={self.num_experts} [{rows}, {cols}] float8_e4m3fn") | |
| self.register_buffer(f"{name}_weight", w) | |
| self.register_buffer(f"{name}_scale", scale) | |
| def weight(self, name): | |
| w, scale = getattr(self, f"{name}_weight"), getattr(self, f"{name}_scale") | |
| out = torch.empty(w.shape, device=w.device, dtype=torch.bfloat16) | |
| step = max(1, self.chunk_bytes // (4 * w[0].numel())) | |
| for e in range(0, self.num_experts, step): | |
| out[e:e + step] = dequant_fp8_block(w[e:e + step], scale[e:e + step]) | |
| return out | |
| def _apply_gate(self, gate_up): | |
| gate, up = gate_up.chunk(2, dim=-1) | |
| return self.act_fn(gate) * up | |
| def forward(self, hidden_states, top_k_index, top_k_weights): | |
| from types import SimpleNamespace | |
| from transformers.integrations.moe import grouped_mm_experts_forward | |
| view = SimpleNamespace(num_experts=self.num_experts, has_gate=True, has_bias=False, is_transposed=False, | |
| gate_up_proj=self.weight("gate_up"), down_proj=self.weight("down"), | |
| _apply_gate=self._apply_gate) | |
| return grouped_mm_experts_forward(view, hidden_states, top_k_index, top_k_weights) | |
| def packed_expert_layers(path, shards, fp8=False): | |
| """Expert prefixes whose every gate/up/down weight is quantized (FP4: ModelOpt uint8 .weight or AutoRound | |
| .weight_packed; fp8: float8 .weight), read from the safetensors headers only. Layers with any BF16 | |
| expert stay unpacked.""" | |
| from safetensors import safe_open | |
| packed, plain = set(), set() | |
| for shard in shards: | |
| with safe_open(str(path / shard), "pt") as st: | |
| for k in st.keys(): | |
| base, attr = k.rsplit(".", 1) | |
| if ".mlp.experts." in k and attr in ("weight", "weight_packed") and base.rsplit(".", 1)[-1] in ("gate_proj", "up_proj", "down_proj"): | |
| prefix = base.rsplit(".", 2)[0] | |
| dt = st.get_slice(k).get_dtype() | |
| is_q = dt == "F8_E4M3" if fp8 else (attr == "weight_packed" or dt == "U8") | |
| (packed if is_q else plain).add(prefix) | |
| return packed - plain | |
| def load_nvfp4(path, device, simulate_activations=False, packed_experts=True, fp8=False): | |
| """ModelOpt or AutoRound NVFP4, or (fp8=True) FP8 block checkpoints, whose experts follow the same | |
| plan with PackedFP8Experts and .weight/.weight_scale_inv (FP8 activations are simulated by the caller). | |
| Build the BF16 backbone directly on the GPU, then fill it shard by shard | |
| (host RAM stays small). The two formats differ only in tensor names and global-scale direction: ModelOpt | |
| .weight/.weight_scale_2/.input_scale (multipliers), AutoRound .weight_packed/.weight_global_scale/ | |
| .input_global_scale (divisors). packed_experts (default, ignored when simulating activations): the FP4 | |
| experts stay packed (PackedNVFP4Experts, ~4.5 bits/weight) and are dequantized one layer at a time inside | |
| forward; the BF16 expert parameters are never allocated. Otherwise every expert is dequantized to BF16 | |
| on load (Clef: ~60 GB instead of ~22 GB).""" | |
| from functools import partial | |
| from safetensors import safe_open | |
| from transformers import AutoConfig, Qwen3OmniMoeForConditionalGeneration | |
| from transformers.models.qwen3_omni_moe import modeling_qwen3_omni_moe as mq | |
| config = AutoConfig.from_pretrained(path) | |
| config.enable_audio_output = False | |
| if simulate_activations: # per-expert F.linear is where the activation rounding hooks in (slow) | |
| for c in (config, config.thinker_config, config.thinker_config.text_config): | |
| c._experts_implementation = "eager" | |
| thinker = getattr(config, "thinker_config", None) | |
| for c in (config, thinker, getattr(thinker, "text_config", None)): # else transformers tries AutoRound | |
| if c is not None and hasattr(c, "quantization_config"): | |
| delattr(c, "quantization_config") | |
| index = json.loads((path / "model.safetensors.index.json").read_text())["weight_map"] \ | |
| if (path / "model.safetensors.index.json").exists() else None | |
| shards = sorted(set(index.values())) if index else ["model.safetensors"] | |
| packable = packed_expert_layers(path, shards, fp8) if packed_experts and not simulate_activations else set() | |
| experts_cls, experts_init = mq.Qwen3OmniMoeThinkerTextExperts, mq.Qwen3OmniMoeThinkerTextExperts.__init__ | |
| if packable: # experts' 3-D BF16 params go to 'meta': no 60 GB transient; materialized below if not packed | |
| def meta_init(self, *a, **k): | |
| with torch.device("meta"): | |
| experts_init(self, *a, **k) | |
| experts_cls.__init__ = meta_init | |
| try: | |
| with torch.device(device): | |
| model = Qwen3OmniMoeForConditionalGeneration._from_config(config, dtype=torch.bfloat16) | |
| finally: | |
| experts_cls.__init__ = experts_init | |
| model.eval() | |
| packed_mods = {} | |
| for name, block in list(model.named_modules()): | |
| if isinstance(getattr(block, "experts", None), experts_cls) and block.experts.gate_up_proj.is_meta: | |
| if f"{name}.experts" in packable: | |
| block.experts = packed_mods[f"{name}.experts"] = PackedFP8Experts(block.experts) if fp8 else \ | |
| PackedNVFP4Experts(block.experts, divisor=None) # NVFP4 multiplier vs divisor: from the tensors | |
| else: | |
| block.experts.to_empty(device=device) | |
| params = dict(model.state_dict()) | |
| filled, experts, in_scales = set(), {}, {} | |
| for shard in shards: | |
| with safe_open(str(path / shard), "pt", device=str(device)) as st: | |
| keys = list(st.keys()) | |
| for k in keys: | |
| base, attr = k.rsplit(".", 1) | |
| if ".mlp.experts." in k and attr in ("weight", "weight_packed") and base.rsplit(".", 1)[-1] in ("gate_proj", "up_proj", "down_proj"): | |
| w = st.get_tensor(k) | |
| prefix, e, proj = base.rsplit(".", 2) # ...mlp.experts, E, gate_proj | |
| if prefix in packed_mods and fp8: # keep FP8: (e4m3 weight, fp32 tile scale_inv) | |
| experts.setdefault(prefix, {})[(int(e), proj)] = (w, st.get_tensor(base + ".weight_scale_inv")) | |
| continue | |
| if prefix in packed_mods: # keep packed: (codes, e4m3 block scale, global scale) | |
| ar = attr == "weight_packed" | |
| mod = packed_mods[prefix] | |
| if mod.divisor is not None and mod.divisor != ar: | |
| raise RuntimeError(f"{prefix}: mixed ModelOpt / AutoRound expert tensors") | |
| mod.divisor = ar | |
| experts.setdefault(prefix, {})[(int(e), proj)] = ( | |
| w, st.get_tensor(base + ".weight_scale"), | |
| st.get_tensor(base + (".weight_global_scale" if ar else ".weight_scale_2"))) | |
| continue | |
| if w.dtype == torch.float8_e4m3fn: | |
| w = dequant_fp8_block(w, st.get_tensor(base + ".weight_scale_inv")) | |
| elif attr == "weight_packed": | |
| w = dequant_nvfp4_ar(w, st.get_tensor(base + ".weight_scale"), st.get_tensor(base + ".weight_global_scale")) | |
| elif w.dtype == torch.uint8: | |
| w = dequant_nvfp4(w, st.get_tensor(base + ".weight_scale"), st.get_tensor(base + ".weight_scale_2")) | |
| experts.setdefault(prefix, {})[(int(e), proj)] = w.to(torch.bfloat16) | |
| elif ".mlp.experts." in k and attr in ("input_scale", "input_global_scale"): | |
| prefix, e, proj = base.rsplit(".", 2) | |
| fq = fp4_fake_quant if attr == "input_scale" else fp4_fake_quant_ar | |
| sc = st.get_tensor(k).float() | |
| got = in_scales.setdefault(prefix, {}).setdefault((int(e), "down" if proj == "down_proj" else "gate_up"), (fq, sc)) | |
| if not torch.equal(got[1], sc): # the fused gate_up_proj can take only one | |
| raise RuntimeError(f"{base}: gate_proj and up_proj input scales differ") | |
| elif attr in ("weight_scale", "weight_scale_2", "input_scale", "weight_global_scale", "input_global_scale", | |
| "weight_scale_inv"): | |
| continue | |
| elif k in params: | |
| w = st.get_tensor(k) | |
| if w.dtype == torch.float8_e4m3fn: # a non-expert FP8 linear (e.g. Qwen-style attention): to BF16 | |
| w = dequant_fp8_block(w, st.get_tensor(base + ".weight_scale_inv")) | |
| params[k].copy_(w) | |
| filled.add(k) | |
| else: | |
| raise RuntimeError(f"checkpoint tensor {k} has no home in the model") | |
| # fuse completed expert layers into the transformers-5 3-D parameters as we go | |
| for prefix in list(experts): | |
| got = experts[prefix] | |
| if prefix in packed_mods: | |
| mod = packed_mods[prefix] | |
| if len(got) < 3 * mod.num_experts: | |
| continue | |
| n = range(mod.num_experts) | |
| mod.set_weights("gate_up", [(got[(e, "gate_proj")], got[(e, "up_proj")]) for e in n]) | |
| mod.set_weights("down", [(got[(e, "down_proj")],) for e in n]) | |
| del experts[prefix] | |
| continue | |
| gu, dn = params[prefix + ".gate_up_proj"], params[prefix + ".down_proj"] | |
| n_exp, inter = gu.shape[0], gu.shape[1] // 2 | |
| if len(got) < 3 * n_exp: | |
| continue | |
| for e in range(n_exp): | |
| gu[e, :inter].copy_(got[(e, "gate_proj")]) | |
| gu[e, inter:].copy_(got[(e, "up_proj")]) | |
| dn[e].copy_(got[(e, "down_proj")]) | |
| filled.update({prefix + ".gate_up_proj", prefix + ".down_proj"}) | |
| del experts[prefix] | |
| torch.cuda.empty_cache() if torch.cuda.is_available() else None | |
| log(f"loaded {shard}") | |
| missing = sorted(set(params) - filled) | |
| if experts or missing: | |
| raise RuntimeError(f"incomplete load: {len(missing)} params unfilled (e.g. {missing[:3]}), " | |
| f"{len(experts)} expert layers incomplete") | |
| if packed_mods: | |
| nbytes = sum(b.numel() * b.element_size() for m in packed_mods.values() for b in m.buffers()) | |
| log(f"packed {'FP8' if fp8 else 'NVFP4'} experts: {len(packed_mods)} layers (weights {nbytes / 2**30:.2f} GiB)") | |
| if simulate_activations: | |
| if not in_scales: | |
| raise RuntimeError("no expert input scale tensors: this checkpoint has no FP4 activations to simulate") | |
| modules = dict(model.named_modules()) | |
| for prefix, sc in in_scales.items(): | |
| fns = {part: [partial(sc[(e, part)][0], global_scale=sc[(e, part)][1].to(device)) | |
| for e in range(modules[prefix].num_experts)] for part in ("gate_up", "down")} | |
| simulate_fp4_activations(modules[prefix], fns["gate_up"], fns["down"]) | |
| log(f"simulating NVFP4 activations in {len(in_scales)} expert layers") | |
| return model | |
| def dequant_int4(qweight, scales, qzeros, g_idx): | |
| """AutoRound/GPTQ int4 (qweight [in/8, out], 8 codes per int32 low bits first, along `in`; qzeros | |
| [groups, out/8] packed along `out`): W[in, out] = scale * (q - (zp + 1)), in BF16. Triton on CUDA | |
| (AutoRound's own tritonv2_zp kernel, fp32 math); pure PyTorch elsewhere (CPU tests, ROCm safety).""" | |
| if qweight.is_cuda: | |
| try: | |
| from auto_round_extension.triton.triton_utils_zp.dequant import dequant248 | |
| return dequant248(qweight, scales, qzeros, g_idx, 4, input_dtype=torch.bfloat16) | |
| except ImportError: | |
| pass | |
| shifts = torch.arange(0, 32, 4, device=qweight.device, dtype=torch.int32) | |
| out = torch.empty(qweight.shape[0] * 8, qweight.shape[1], device=qweight.device, dtype=torch.bfloat16) | |
| rows = 8 * 128 * 16 # bounded transients: 16 groups of 128 input rows at a time | |
| for r in range(0, out.shape[0], rows): | |
| q = ((qweight[r // 8:(r + rows) // 8].unsqueeze(1) >> shifts[:, None]) & 15).flatten(0, 1) | |
| g = g_idx[r:r + rows].long() | |
| z = ((qzeros[g].unsqueeze(-1) >> shifts) & 15).flatten(1) + 1 | |
| out[r:r + rows] = (scales[g].float() * (q - z).float()).to(torch.bfloat16) | |
| return out | |
| class PackedInt4Experts(torch.nn.Module): | |
| """All experts of one MoE layer as two stacked int4 matrices (gate|up, down), experts stacked along | |
| the input dim so one dequant yields the [E, in, out] BF16 layout transformers' grouped_mm wants.""" | |
| def __init__(self, experts, act_fn): | |
| super().__init__() | |
| self.num_experts, self.act_fn = len(experts), act_fn | |
| for name, parts in (("gate_up", [(e.gate_proj, e.up_proj) for e in experts]), | |
| ("down", [(e.down_proj,) for e in experts])): | |
| qw = torch.cat([torch.cat([p.qweight for p in ps], 1) for ps in parts], 0) | |
| qz = torch.cat([torch.cat([p.qzeros for p in ps], 1) for ps in parts], 0) | |
| sc = torch.cat([torch.cat([p.scales for p in ps], 1) for ps in parts], 0) | |
| self.register_buffer(f"{name}_qweight", qw) | |
| self.register_buffer(f"{name}_qzeros", qz) | |
| self.register_buffer(f"{name}_scales", sc) | |
| self.register_buffer(f"{name}_g_idx", torch.arange(qw.shape[0] * 8, device=qw.device, dtype=torch.int32) // 128) | |
| setattr(self, f"{name}_in", parts[0][0].infeatures) | |
| def weight(self, name): | |
| w = dequant_int4(*(getattr(self, f"{name}_{t}") for t in ("qweight", "scales", "qzeros", "g_idx"))) | |
| return w.view(self.num_experts, getattr(self, f"{name}_in"), -1) | |
| def _apply_gate(self, gate_up): | |
| gate, up = gate_up.chunk(2, dim=-1) | |
| return self.act_fn(gate) * up | |
| def forward(self, hidden_states, top_k_index, top_k_weights): | |
| """transformers' grouped_mm experts forward over per-layer BF16 temporaries (freed on return).""" | |
| from types import SimpleNamespace | |
| from transformers.integrations.moe import grouped_mm_experts_forward | |
| view = SimpleNamespace(num_experts=self.num_experts, has_gate=True, has_bias=False, is_transposed=True, | |
| gate_up_proj=self.weight("gate_up"), down_proj=self.weight("down"), | |
| _apply_gate=self._apply_gate) | |
| return grouped_mm_experts_forward(view, hidden_states, top_k_index, top_k_weights) | |
| def fast_int4_moe(model): | |
| """Swap AutoRound's per-expert QuantLinear loop (~2.7 s/forward on Clef) for one dequant + grouped_mm | |
| per layer. Weights stay packed int4; only one layer's BF16 experts exist at a time. Anything | |
| unexpected: warn and keep the slow path for that layer.""" | |
| done = 0 | |
| for name, block in list(model.named_modules()): | |
| experts = getattr(block, "experts", None) | |
| if not (type(experts).__name__ == "SequentialQwen3OmniThinkerExperts" and callable(getattr(block, "experts_forward", None))): | |
| continue | |
| try: | |
| projs = [getattr(e, p) for e in experts for p in ("gate_proj", "up_proj", "down_proj")] | |
| for p in projs: | |
| gs, inf, outf = p.group_size, p.infeatures, p.outfeatures | |
| ok = (p.bits == 4 and gs == 128 and inf % gs == 0 and p.bias is None | |
| and not getattr(p, "use_generic_bit_packing", False) | |
| and p.qweight.dtype == p.qzeros.dtype == torch.int32 | |
| and tuple(p.qweight.shape) == (inf // 8, outf) and tuple(p.qzeros.shape) == (inf // gs, outf // 8) | |
| and tuple(p.scales.shape) == (inf // gs, outf)) | |
| g_idx = getattr(p, "g_idx", None) | |
| ok = ok and (g_idx is None or torch.equal(g_idx.long().cpu(), torch.arange(inf) // gs)) | |
| if not ok: | |
| raise ValueError(f"unsupported expert projection {type(p).__name__} " | |
| f"(bits={p.bits}, group_size={gs}, qweight {tuple(p.qweight.shape)})") | |
| if len({(e.gate_proj.infeatures, e.gate_proj.outfeatures, e.up_proj.outfeatures, e.down_proj.outfeatures) | |
| for e in experts}) != 1 or experts[0].gate_proj.outfeatures != experts[0].down_proj.infeatures: | |
| raise ValueError("experts differ in shape") | |
| packed = PackedInt4Experts(experts, experts.act_fn) | |
| except Exception as e: | |
| log(f"WARNING: fast int4 MoE skipped for {name}: {e}") | |
| continue | |
| block.experts = packed # drops the per-expert QuantLinears | |
| block.experts_forward = packed.__call__ # the block calls experts_forward(h, top_k_index, top_k_weights) | |
| done += 1 | |
| log(f"fast grouped int4 MoE: {done} layers") | |
| return done | |
| def load(path, device, simulate_fp4_activations=None, fast_moe=True, packed_nvfp4=True, packed_fp8=True, | |
| simulate_fp8_activations=False): | |
| """simulate_fp4_activations: off unless asked. On W4A4 checkpoints it reproduces the quantized model's | |
| answers exactly (for benchmarking), at a large speed cost (eager per-expert path). | |
| fast_moe: AutoRound INT4 g128 checkpoints run experts as dequant + grouped_mm (see fast_int4_moe). | |
| packed_nvfp4: NVFP4 checkpoints keep experts packed, dequantized per layer in forward (PackedNVFP4Experts). | |
| packed_fp8 / simulate_fp8_activations: the same for FP8 block checkpoints (PackedFP8Experts), plus | |
| optional dynamic FP8 rounding of expert inputs (cheap: no eager path needed).""" | |
| sys.path.insert(0, str(path)) | |
| import joint_schema_model as jsm | |
| kind = kind_of(path) | |
| log(f"loading {path} ({kind}) on {device}") | |
| if kind == "bf16": | |
| model, processor = jsm.load_release_model(path, device=device) | |
| return model, processor, jsm, kind | |
| from safetensors.torch import load_file | |
| from transformers import AutoConfig, AutoProcessor, Qwen3OmniMoeForConditionalGeneration | |
| if kind.startswith(("modelopt:", "auto-round:nvfp4")): | |
| simulate_fp4_activations = bool(simulate_fp4_activations) and kind in ("modelopt:NVFP4", "auto-round:nvfp4") | |
| backbone = load_nvfp4(path, device, simulate_fp4_activations, packed_nvfp4) | |
| kind += "+fp4-activations" if simulate_fp4_activations else "" | |
| kind += "+packed-experts" if any(isinstance(m, PackedNVFP4Experts) for m in backbone.modules()) else "" | |
| elif kind == "fp8-block": | |
| backbone = load_nvfp4(path, device, False, packed_fp8, fp8=True) | |
| if simulate_fp8_activations: | |
| from transformers.models.qwen3_omni_moe import modeling_qwen3_omni_moe as mq | |
| layers = [m for m in backbone.thinker.modules() if isinstance(m, (PackedFP8Experts, mq.Qwen3OmniMoeThinkerTextExperts))] | |
| for m in layers: | |
| round_fp8_expert_inputs(m) | |
| log(f"simulating dynamic FP8 activations in {len(layers)} expert layers") | |
| kind += "+fp8-activations" | |
| kind += "+packed-experts" if any(isinstance(m, PackedFP8Experts) for m in backbone.modules()) else "" | |
| else: | |
| config = AutoConfig.from_pretrained(path) | |
| config.enable_audio_output = False | |
| backbone = Qwen3OmniMoeForConditionalGeneration.from_pretrained( | |
| path, config=config, dtype=torch.bfloat16, device_map={"": str(device)}) | |
| qc = config.quantization_config if isinstance(config.quantization_config, dict) else config.quantization_config.to_dict() | |
| if fast_moe and kind == "auto-round:4bit" and qc.get("group_size") == 128 \ | |
| and "gptq" in str(qc.get("packing_format", "auto_round:auto_gptq")): | |
| kind += "+grouped-moe" if fast_int4_moe(backbone) else "" | |
| torch.cuda.empty_cache() if torch.cuda.is_available() else None | |
| head = jsm.JointSchemaHead(**json.loads((path / "joint_head_config.json").read_text())) | |
| head.load_state_dict(load_file(path / "joint_head.safetensors"), strict=True) | |
| head = head.to(device=device, dtype=torch.bfloat16) | |
| model = jsm.ClefModel(backbone.thinker, head).eval() | |
| return model, AutoProcessor.from_pretrained(path), jsm, kind | |
| DEMO = { | |
| "model": "clef-omni", | |
| "state": {"goal": "Choose the next game action", "observation": "The player is facing a wall."}, | |
| "questions": {"action": {"type": "choice", "instructions": "What is a sensible next action?", | |
| "criteria": {"forward": "Move forward", "left": "Turn left", "right": "Turn right", "stop": "Stop"}}, | |
| "safe": {"type": "noul", "instructions": "Is it safe to keep moving forward?"}}, | |
| } | |
| def main(): | |
| ap = argparse.ArgumentParser(prog="serve_clef.py", description=__doc__.split("\n\n")[0]) | |
| ap.add_argument("--model", default="dbirks/clef-omni-nvfp4", help="HF repo id or local directory") | |
| ap.add_argument("--revision", default=None) | |
| ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu") | |
| ap.add_argument("--host", default="127.0.0.1") | |
| ap.add_argument("--port", type=int, default=8000) | |
| ap.add_argument("--demo", action="store_true", help="answer one built-in request and exit") | |
| ap.add_argument("--request", help="answer the JSON request in this file and exit") | |
| ap.add_argument("--simulate-fp4-activations", action=argparse.BooleanOptionalAction, default=False, | |
| help="NVFP4 W4A4 checkpoints: also round expert inputs to FP4, reproducing the quantized " | |
| "model's answers exactly (slow; for benchmarking). Default: FP4 weights, BF16 activations") | |
| ap.add_argument("--no-fast-moe", dest="fast_moe", action="store_false", | |
| help="AutoRound INT4: keep AutoRound's per-expert QuantLinear loop instead of dequant + grouped_mm") | |
| ap.add_argument("--no-packed-nvfp4", dest="packed_nvfp4", action="store_false", | |
| help="NVFP4: dequantize every expert to BF16 on load (~60 GB on Clef) instead of keeping them " | |
| "packed (~22 GB) and dequantizing one layer at a time in forward") | |
| ap.add_argument("--no-packed-fp8", dest="packed_fp8", action="store_false", | |
| help="FP8 block: dequantize every expert to BF16 on load (~60 GB on Clef) instead of keeping " | |
| "them FP8 (~32 GB) and dequantizing one layer at a time in forward") | |
| ap.add_argument("--simulate-fp8-activations", action=argparse.BooleanOptionalAction, default=False, | |
| help="FP8 block: also round expert inputs to FP8 per 1x128 token group (W8A8-dynamic, as native " | |
| "FP8 kernels run it). Default: FP8 weights, BF16 activations") | |
| args = ap.parse_args() | |
| path = resolve(args.model, args.revision) | |
| if args.demo or args.request: | |
| t0 = time.time() | |
| model, processor, jsm, kind = load(path, args.device, args.simulate_fp4_activations, args.fast_moe, | |
| args.packed_nvfp4, args.packed_fp8, args.simulate_fp8_activations) | |
| log(f"ready in {time.time() - t0:.0f}s") | |
| req = DEMO if args.demo else json.loads(Path(args.request).read_text()) | |
| print(json.dumps(jsm.systemone(model, processor, req), indent=2)) | |
| return | |
| import uvicorn | |
| from fastapi import FastAPI, HTTPException, Request | |
| from fastapi.responses import JSONResponse | |
| state = {"ready": False, "requests": 0, "model_id": args.model} | |
| lock = threading.Lock() | |
| def _load(): | |
| t0 = time.time() | |
| model, processor, jsm, kind = load(path, args.device, args.simulate_fp4_activations, args.fast_moe, | |
| args.packed_nvfp4, args.packed_fp8, args.simulate_fp8_activations) | |
| state.update(model=model, processor=processor, jsm=jsm, kind=kind, ready=True, load_s=round(time.time() - t0, 1)) | |
| log(f"ready in {state['load_s']}s") | |
| app = FastAPI() | |
| def healthz(): | |
| body = {k: v for k, v in state.items() if k not in ("model", "processor", "jsm")} | |
| return JSONResponse(body, status_code=200 if state["ready"] else 503) | |
| def systemone(body: dict): | |
| if not state["ready"]: | |
| raise HTTPException(503, "model loading") | |
| with lock: # one GPU: serve one decision at a time | |
| state["requests"] += 1 | |
| try: | |
| return state["jsm"].systemone(state["model"], state["processor"], body) | |
| except torch.OutOfMemoryError as e: | |
| torch.cuda.empty_cache() | |
| raise HTTPException(422, f"too many tokens: CUDA out of memory ({str(e).splitlines()[0][:200]})") | |
| except ValueError as e: | |
| msg = str(e) | |
| raise HTTPException(422, f"maximum context length: {msg}" if "maximum is" in msg else msg) | |
| threading.Thread(target=_load, daemon=True).start() | |
| uvicorn.run(app, host=args.host, port=args.port, log_level="info") | |
| if __name__ == "__main__": | |
| main() | |