Text Generation
Transformers
Safetensors
English
Chinese
Yue Chinese
volundr
image-text-to-text
agens
blockway
agent
tool-use
code
long-context
linear-attention
sparse-attention
cantonese
conversational
custom_code
Instructions to use Blockway/Agens-Volundr-32B-Preview with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Blockway/Agens-Volundr-32B-Preview with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="Blockway/Agens-Volundr-32B-Preview", trust_remote_code=True) messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForImageTextToText model = AutoModelForImageTextToText.from_pretrained("Blockway/Agens-Volundr-32B-Preview", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Blockway/Agens-Volundr-32B-Preview with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Blockway/Agens-Volundr-32B-Preview" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Blockway/Agens-Volundr-32B-Preview", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/Blockway/Agens-Volundr-32B-Preview
- SGLang
How to use Blockway/Agens-Volundr-32B-Preview with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Blockway/Agens-Volundr-32B-Preview" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Blockway/Agens-Volundr-32B-Preview", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Blockway/Agens-Volundr-32B-Preview" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Blockway/Agens-Volundr-32B-Preview", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use Blockway/Agens-Volundr-32B-Preview with Docker Model Runner:
docker model run hf.co/Blockway/Agens-Volundr-32B-Preview
Blockway commited on
add bcsa.py
Browse files
bcsa.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""BCSA — Blockway Compressed-Sparse Attention (training-time reference implementation + cached decode).
|
| 2 |
+
|
| 3 |
+
Per query position t (causal):
|
| 4 |
+
* local branch : dense attention over keys in (t - W, t] (W = bcsa_window, default 4096)
|
| 5 |
+
* far branch : keys/values older than the window are mean-pooled in blocks of C positions (C = bcsa_compress);
|
| 6 |
+
a light indexer (Hq query heads x 1 key head, dim Di) scores every compressed block and the
|
| 7 |
+
top-k blocks are attended to; softmax is shared across local + selected far keys.
|
| 8 |
+
Projections / q,k norms / RoPE / sigmoid output gate / o_proj are Qwen3.5's (transplanted verbatim).
|
| 9 |
+
Step-0 exactness: for sequences <= W the far branch is empty -> output == Qwen3.5 dense attention exactly.
|
| 10 |
+
Indexer is the only new parameter group (trained by KL warm-up vs dense attention scores, backbone frozen).
|
| 11 |
+
This reference materialises dense scores + masks (O(T^2/C) for the far branch) — fine for training at <=32K;
|
| 12 |
+
`bcsa_flex.py` (flex_attention) replaces it for training at 8K+; incremental decode (past_key_values) uses this
|
| 13 |
+
reference path over the cached K/V with absolute query positions (T_q small).
|
| 14 |
+
"""
|
| 15 |
+
import math
|
| 16 |
+
import torch, torch.nn as nn, torch.nn.functional as F
|
| 17 |
+
from transformers.models.qwen3_5 import modeling_qwen3_5 as Q
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _pool_blocks(x, C):
|
| 21 |
+
"""x: [B, H, T, D] -> mean-pooled blocks [B, H, T//C, D] (drops the tail remainder)."""
|
| 22 |
+
B, H, T, D = x.shape
|
| 23 |
+
n = T // C
|
| 24 |
+
if n == 0:
|
| 25 |
+
return x[:, :, :0]
|
| 26 |
+
return x[:, :, : n * C].reshape(B, H, n, C, D).mean(dim=3)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class BCSAAttention(nn.Module):
|
| 30 |
+
def __init__(self, config, layer_idx: int, indexer_heads: int = 4, indexer_dim: int = 128):
|
| 31 |
+
super().__init__()
|
| 32 |
+
self.config = config; self.layer_idx = layer_idx
|
| 33 |
+
self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
|
| 34 |
+
self.n_heads = config.num_attention_heads; self.n_kv = config.num_key_value_heads
|
| 35 |
+
self.n_groups = self.n_heads // self.n_kv
|
| 36 |
+
self.scaling = self.head_dim ** -0.5
|
| 37 |
+
self.window = config.bcsa_window; self.C = config.bcsa_compress; self.topk = config.bcsa_topk
|
| 38 |
+
# --- transplanted (same names as Qwen3_5Attention) ---
|
| 39 |
+
self.q_proj = nn.Linear(config.hidden_size, self.n_heads * self.head_dim * 2, bias=config.attention_bias)
|
| 40 |
+
self.k_proj = nn.Linear(config.hidden_size, self.n_kv * self.head_dim, bias=config.attention_bias)
|
| 41 |
+
self.v_proj = nn.Linear(config.hidden_size, self.n_kv * self.head_dim, bias=config.attention_bias)
|
| 42 |
+
self.o_proj = nn.Linear(self.n_heads * self.head_dim, config.hidden_size, bias=config.attention_bias)
|
| 43 |
+
self.q_norm = Q.Qwen3_5RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
| 44 |
+
self.k_norm = Q.Qwen3_5RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
| 45 |
+
# --- new: block indexer (Blockway) ---
|
| 46 |
+
self.idx_heads, self.idx_dim = indexer_heads, indexer_dim
|
| 47 |
+
self.idx_q = nn.Linear(config.hidden_size, indexer_heads * indexer_dim, bias=False)
|
| 48 |
+
self.idx_k = nn.Linear(config.hidden_size, indexer_dim, bias=False)
|
| 49 |
+
self.idx_norm = Q.Qwen3_5RMSNorm(indexer_dim, eps=config.rms_norm_eps)
|
| 50 |
+
self.is_causal = True
|
| 51 |
+
self._ki_cache = {} # id(past_key_values) -> cached indexer keys [B, T_kv, Di] (decode only; not a parameter)
|
| 52 |
+
|
| 53 |
+
# ------------------------------------------------------------------ helpers
|
| 54 |
+
def _qkv(self, hidden_states, position_embeddings):
|
| 55 |
+
B, T, _ = hidden_states.shape
|
| 56 |
+
q, gate = torch.chunk(self.q_proj(hidden_states).view(B, T, -1, self.head_dim * 2), 2, dim=-1)
|
| 57 |
+
gate = gate.reshape(B, T, -1)
|
| 58 |
+
q = self.q_norm(q.reshape(B, T, self.n_heads, self.head_dim)).transpose(1, 2) # [B,H,T,D]
|
| 59 |
+
k = self.k_norm(self.k_proj(hidden_states).view(B, T, self.n_kv, self.head_dim)).transpose(1, 2)
|
| 60 |
+
v = self.v_proj(hidden_states).view(B, T, self.n_kv, self.head_dim).transpose(1, 2)
|
| 61 |
+
cos, sin = position_embeddings
|
| 62 |
+
q, k = Q.apply_rotary_pos_emb(q, k, cos, sin)
|
| 63 |
+
return q, k, v, gate
|
| 64 |
+
|
| 65 |
+
def indexer_keys(self, hidden_states):
|
| 66 |
+
return self.idx_norm(self.idx_k(hidden_states)) # [B,T,Di]
|
| 67 |
+
|
| 68 |
+
def indexer_scores(self, hidden_states, T, ki_full=None):
|
| 69 |
+
"""Block relevance logits [B, T_q, nblocks] (mean over indexer heads). ki_full: cached keys for all kv positions."""
|
| 70 |
+
B = hidden_states.shape[0]
|
| 71 |
+
qi = self.idx_norm(self.idx_q(hidden_states).view(B, T, self.idx_heads, self.idx_dim)) # [B,T,Hq,Di]
|
| 72 |
+
ki = self.indexer_keys(hidden_states) if ki_full is None else ki_full # [B,T_kv,Di]
|
| 73 |
+
kb = _pool_blocks(ki[:, None], self.C)[:, 0] # [B,nb,Di]
|
| 74 |
+
s = torch.einsum("bthd,bnd->bthn", qi, kb).mean(2) * (self.idx_dim ** -0.5) # [B,T,nb]
|
| 75 |
+
return s
|
| 76 |
+
|
| 77 |
+
def _far_mask(self, T, device, q_pos=None):
|
| 78 |
+
"""valid[t, n] = block n lies entirely before t's local window: (n+1)*C - 1 <= t - W (causal & disjoint).
|
| 79 |
+
T = number of kv positions; q_pos = absolute query positions (default 0..T-1)."""
|
| 80 |
+
nb = T // self.C
|
| 81 |
+
t = (torch.arange(T, device=device) if q_pos is None else q_pos)[:, None]; n = torch.arange(nb, device=device)[None, :]
|
| 82 |
+
return ((n + 1) * self.C - 1) <= (t - self.window) # [T_q,nb] bool
|
| 83 |
+
|
| 84 |
+
# ------------------------------------------------------------------ forward
|
| 85 |
+
def forward(self, hidden_states, position_embeddings, attention_mask=None, past_key_values=None,
|
| 86 |
+
cache_position=None, return_indexer_logits: bool = False, **kwargs):
|
| 87 |
+
B, T, _ = hidden_states.shape
|
| 88 |
+
q, k, v, gate = self._qkv(hidden_states, position_embeddings)
|
| 89 |
+
dev = q.device
|
| 90 |
+
# ---- KV cache (incremental decode): cached keys are post-RoPE, like Qwen3.5 ----
|
| 91 |
+
ki_full = None; prev = 0
|
| 92 |
+
if past_key_values is not None:
|
| 93 |
+
prev = past_key_values.get_seq_length(self.layer_idx)
|
| 94 |
+
k, v = past_key_values.update(k, v, self.layer_idx, {"cache_position": cache_position}) # [B,Hkv,T_kv,D]
|
| 95 |
+
ki_new = self.indexer_keys(hidden_states)
|
| 96 |
+
key = id(past_key_values)
|
| 97 |
+
ki_full = ki_new if prev == 0 or key not in self._ki_cache else torch.cat([self._ki_cache[key], ki_new], dim=1)
|
| 98 |
+
self._ki_cache[key] = ki_full
|
| 99 |
+
T_kv = k.shape[2]
|
| 100 |
+
q_pos = torch.arange(prev, prev + T, device=dev) # absolute query positions
|
| 101 |
+
kf = k.repeat_interleave(self.n_groups, dim=1); vf = v.repeat_interleave(self.n_groups, dim=1) # [B,H,T_kv,D]
|
| 102 |
+
# ---- local (sliding-window causal) scores ----
|
| 103 |
+
s_loc = torch.matmul(q, kf.transpose(2, 3)) * self.scaling # [B,H,T,T_kv]
|
| 104 |
+
kv_idx = torch.arange(T_kv, device=dev)
|
| 105 |
+
loc_ok = (kv_idx[None, :] <= q_pos[:, None]) & (kv_idx[None, :] > (q_pos[:, None] - self.window))
|
| 106 |
+
s_loc = s_loc.masked_fill(~loc_ok, float("-inf"))
|
| 107 |
+
# ---- far (compressed blocks + top-k) ----
|
| 108 |
+
nb = T_kv // self.C
|
| 109 |
+
far_valid = self._far_mask(T_kv, dev, q_pos) if nb > 0 else None
|
| 110 |
+
if nb > 0 and far_valid.any():
|
| 111 |
+
kb = _pool_blocks(kf, self.C); vb = _pool_blocks(vf, self.C) # [B,H,nb,D]
|
| 112 |
+
s_far = torch.matmul(q, kb.transpose(2, 3)) * self.scaling # [B,H,T,nb]
|
| 113 |
+
idx = self.indexer_scores(hidden_states, T, ki_full) # [B,T,nb]
|
| 114 |
+
idx = idx.masked_fill(~far_valid[None], float("-inf"))
|
| 115 |
+
k_sel = min(self.topk, nb)
|
| 116 |
+
top = idx.topk(k_sel, dim=-1).indices # [B,T,k]
|
| 117 |
+
sel = torch.zeros_like(idx, dtype=torch.bool).scatter_(-1, top, True) & far_valid[None] # [B,T,nb]
|
| 118 |
+
s_far = s_far.masked_fill(~sel[:, None], float("-inf"))
|
| 119 |
+
scores = torch.cat([s_loc, s_far], dim=-1)
|
| 120 |
+
vals = torch.cat([vf, vb], dim=2)
|
| 121 |
+
else:
|
| 122 |
+
scores, vals, idx = s_loc, vf, None
|
| 123 |
+
p = F.softmax(scores.float(), dim=-1).to(q.dtype)
|
| 124 |
+
out = torch.matmul(p, vals).transpose(1, 2).reshape(B, T, -1)
|
| 125 |
+
out = out * torch.sigmoid(gate)
|
| 126 |
+
out = self.o_proj(out)
|
| 127 |
+
if return_indexer_logits:
|
| 128 |
+
return out, idx
|
| 129 |
+
return out, None
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def indexer_kl_target(q, kb, far_valid, scaling):
|
| 133 |
+
"""Teacher for the indexer warm-up: dense attention's block-level relevance (softmax over valid far blocks of the
|
| 134 |
+
max-over-heads score), detached. q: [B,H,T,D], kb: [B,H,nb,D] -> [B,T,nb] probabilities."""
|
| 135 |
+
with torch.no_grad():
|
| 136 |
+
s = (torch.matmul(q, kb.transpose(2, 3)) * scaling).amax(dim=1) # [B,T,nb]
|
| 137 |
+
s = s.masked_fill(~far_valid[None], float("-inf"))
|
| 138 |
+
return F.softmax(s.float(), dim=-1)
|