Blockway commited on
Commit
c5812cd
·
verified ·
1 Parent(s): 324514f

add bcsa.py

Browse files
Files changed (1) hide show
  1. bcsa.py +138 -0
bcsa.py ADDED
@@ -0,0 +1,138 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """BCSA — Blockway Compressed-Sparse Attention (training-time reference implementation + cached decode).
2
+
3
+ Per query position t (causal):
4
+ * local branch : dense attention over keys in (t - W, t] (W = bcsa_window, default 4096)
5
+ * far branch : keys/values older than the window are mean-pooled in blocks of C positions (C = bcsa_compress);
6
+ a light indexer (Hq query heads x 1 key head, dim Di) scores every compressed block and the
7
+ top-k blocks are attended to; softmax is shared across local + selected far keys.
8
+ Projections / q,k norms / RoPE / sigmoid output gate / o_proj are Qwen3.5's (transplanted verbatim).
9
+ Step-0 exactness: for sequences <= W the far branch is empty -> output == Qwen3.5 dense attention exactly.
10
+ Indexer is the only new parameter group (trained by KL warm-up vs dense attention scores, backbone frozen).
11
+ This reference materialises dense scores + masks (O(T^2/C) for the far branch) — fine for training at <=32K;
12
+ `bcsa_flex.py` (flex_attention) replaces it for training at 8K+; incremental decode (past_key_values) uses this
13
+ reference path over the cached K/V with absolute query positions (T_q small).
14
+ """
15
+ import math
16
+ import torch, torch.nn as nn, torch.nn.functional as F
17
+ from transformers.models.qwen3_5 import modeling_qwen3_5 as Q
18
+
19
+
20
+ def _pool_blocks(x, C):
21
+ """x: [B, H, T, D] -> mean-pooled blocks [B, H, T//C, D] (drops the tail remainder)."""
22
+ B, H, T, D = x.shape
23
+ n = T // C
24
+ if n == 0:
25
+ return x[:, :, :0]
26
+ return x[:, :, : n * C].reshape(B, H, n, C, D).mean(dim=3)
27
+
28
+
29
+ class BCSAAttention(nn.Module):
30
+ def __init__(self, config, layer_idx: int, indexer_heads: int = 4, indexer_dim: int = 128):
31
+ super().__init__()
32
+ self.config = config; self.layer_idx = layer_idx
33
+ self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
34
+ self.n_heads = config.num_attention_heads; self.n_kv = config.num_key_value_heads
35
+ self.n_groups = self.n_heads // self.n_kv
36
+ self.scaling = self.head_dim ** -0.5
37
+ self.window = config.bcsa_window; self.C = config.bcsa_compress; self.topk = config.bcsa_topk
38
+ # --- transplanted (same names as Qwen3_5Attention) ---
39
+ self.q_proj = nn.Linear(config.hidden_size, self.n_heads * self.head_dim * 2, bias=config.attention_bias)
40
+ self.k_proj = nn.Linear(config.hidden_size, self.n_kv * self.head_dim, bias=config.attention_bias)
41
+ self.v_proj = nn.Linear(config.hidden_size, self.n_kv * self.head_dim, bias=config.attention_bias)
42
+ self.o_proj = nn.Linear(self.n_heads * self.head_dim, config.hidden_size, bias=config.attention_bias)
43
+ self.q_norm = Q.Qwen3_5RMSNorm(self.head_dim, eps=config.rms_norm_eps)
44
+ self.k_norm = Q.Qwen3_5RMSNorm(self.head_dim, eps=config.rms_norm_eps)
45
+ # --- new: block indexer (Blockway) ---
46
+ self.idx_heads, self.idx_dim = indexer_heads, indexer_dim
47
+ self.idx_q = nn.Linear(config.hidden_size, indexer_heads * indexer_dim, bias=False)
48
+ self.idx_k = nn.Linear(config.hidden_size, indexer_dim, bias=False)
49
+ self.idx_norm = Q.Qwen3_5RMSNorm(indexer_dim, eps=config.rms_norm_eps)
50
+ self.is_causal = True
51
+ self._ki_cache = {} # id(past_key_values) -> cached indexer keys [B, T_kv, Di] (decode only; not a parameter)
52
+
53
+ # ------------------------------------------------------------------ helpers
54
+ def _qkv(self, hidden_states, position_embeddings):
55
+ B, T, _ = hidden_states.shape
56
+ q, gate = torch.chunk(self.q_proj(hidden_states).view(B, T, -1, self.head_dim * 2), 2, dim=-1)
57
+ gate = gate.reshape(B, T, -1)
58
+ q = self.q_norm(q.reshape(B, T, self.n_heads, self.head_dim)).transpose(1, 2) # [B,H,T,D]
59
+ k = self.k_norm(self.k_proj(hidden_states).view(B, T, self.n_kv, self.head_dim)).transpose(1, 2)
60
+ v = self.v_proj(hidden_states).view(B, T, self.n_kv, self.head_dim).transpose(1, 2)
61
+ cos, sin = position_embeddings
62
+ q, k = Q.apply_rotary_pos_emb(q, k, cos, sin)
63
+ return q, k, v, gate
64
+
65
+ def indexer_keys(self, hidden_states):
66
+ return self.idx_norm(self.idx_k(hidden_states)) # [B,T,Di]
67
+
68
+ def indexer_scores(self, hidden_states, T, ki_full=None):
69
+ """Block relevance logits [B, T_q, nblocks] (mean over indexer heads). ki_full: cached keys for all kv positions."""
70
+ B = hidden_states.shape[0]
71
+ qi = self.idx_norm(self.idx_q(hidden_states).view(B, T, self.idx_heads, self.idx_dim)) # [B,T,Hq,Di]
72
+ ki = self.indexer_keys(hidden_states) if ki_full is None else ki_full # [B,T_kv,Di]
73
+ kb = _pool_blocks(ki[:, None], self.C)[:, 0] # [B,nb,Di]
74
+ s = torch.einsum("bthd,bnd->bthn", qi, kb).mean(2) * (self.idx_dim ** -0.5) # [B,T,nb]
75
+ return s
76
+
77
+ def _far_mask(self, T, device, q_pos=None):
78
+ """valid[t, n] = block n lies entirely before t's local window: (n+1)*C - 1 <= t - W (causal & disjoint).
79
+ T = number of kv positions; q_pos = absolute query positions (default 0..T-1)."""
80
+ nb = T // self.C
81
+ t = (torch.arange(T, device=device) if q_pos is None else q_pos)[:, None]; n = torch.arange(nb, device=device)[None, :]
82
+ return ((n + 1) * self.C - 1) <= (t - self.window) # [T_q,nb] bool
83
+
84
+ # ------------------------------------------------------------------ forward
85
+ def forward(self, hidden_states, position_embeddings, attention_mask=None, past_key_values=None,
86
+ cache_position=None, return_indexer_logits: bool = False, **kwargs):
87
+ B, T, _ = hidden_states.shape
88
+ q, k, v, gate = self._qkv(hidden_states, position_embeddings)
89
+ dev = q.device
90
+ # ---- KV cache (incremental decode): cached keys are post-RoPE, like Qwen3.5 ----
91
+ ki_full = None; prev = 0
92
+ if past_key_values is not None:
93
+ prev = past_key_values.get_seq_length(self.layer_idx)
94
+ k, v = past_key_values.update(k, v, self.layer_idx, {"cache_position": cache_position}) # [B,Hkv,T_kv,D]
95
+ ki_new = self.indexer_keys(hidden_states)
96
+ key = id(past_key_values)
97
+ ki_full = ki_new if prev == 0 or key not in self._ki_cache else torch.cat([self._ki_cache[key], ki_new], dim=1)
98
+ self._ki_cache[key] = ki_full
99
+ T_kv = k.shape[2]
100
+ q_pos = torch.arange(prev, prev + T, device=dev) # absolute query positions
101
+ kf = k.repeat_interleave(self.n_groups, dim=1); vf = v.repeat_interleave(self.n_groups, dim=1) # [B,H,T_kv,D]
102
+ # ---- local (sliding-window causal) scores ----
103
+ s_loc = torch.matmul(q, kf.transpose(2, 3)) * self.scaling # [B,H,T,T_kv]
104
+ kv_idx = torch.arange(T_kv, device=dev)
105
+ loc_ok = (kv_idx[None, :] <= q_pos[:, None]) & (kv_idx[None, :] > (q_pos[:, None] - self.window))
106
+ s_loc = s_loc.masked_fill(~loc_ok, float("-inf"))
107
+ # ---- far (compressed blocks + top-k) ----
108
+ nb = T_kv // self.C
109
+ far_valid = self._far_mask(T_kv, dev, q_pos) if nb > 0 else None
110
+ if nb > 0 and far_valid.any():
111
+ kb = _pool_blocks(kf, self.C); vb = _pool_blocks(vf, self.C) # [B,H,nb,D]
112
+ s_far = torch.matmul(q, kb.transpose(2, 3)) * self.scaling # [B,H,T,nb]
113
+ idx = self.indexer_scores(hidden_states, T, ki_full) # [B,T,nb]
114
+ idx = idx.masked_fill(~far_valid[None], float("-inf"))
115
+ k_sel = min(self.topk, nb)
116
+ top = idx.topk(k_sel, dim=-1).indices # [B,T,k]
117
+ sel = torch.zeros_like(idx, dtype=torch.bool).scatter_(-1, top, True) & far_valid[None] # [B,T,nb]
118
+ s_far = s_far.masked_fill(~sel[:, None], float("-inf"))
119
+ scores = torch.cat([s_loc, s_far], dim=-1)
120
+ vals = torch.cat([vf, vb], dim=2)
121
+ else:
122
+ scores, vals, idx = s_loc, vf, None
123
+ p = F.softmax(scores.float(), dim=-1).to(q.dtype)
124
+ out = torch.matmul(p, vals).transpose(1, 2).reshape(B, T, -1)
125
+ out = out * torch.sigmoid(gate)
126
+ out = self.o_proj(out)
127
+ if return_indexer_logits:
128
+ return out, idx
129
+ return out, None
130
+
131
+
132
+ def indexer_kl_target(q, kb, far_valid, scaling):
133
+ """Teacher for the indexer warm-up: dense attention's block-level relevance (softmax over valid far blocks of the
134
+ max-over-heads score), detached. q: [B,H,T,D], kb: [B,H,nb,D] -> [B,T,nb] probabilities."""
135
+ with torch.no_grad():
136
+ s = (torch.matmul(q, kb.transpose(2, 3)) * scaling).amax(dim=1) # [B,T,nb]
137
+ s = s.masked_fill(~far_valid[None], float("-inf"))
138
+ return F.softmax(s.float(), dim=-1)