bghira commited on
Commit
d91ccd0
·
verified ·
1 Parent(s): fe77096

Publish calibrated CLEF INT8 ConvRot text-linear overrides

Browse files
Files changed (37) hide show
  1. LICENSE +202 -0
  2. NOTICE +20 -0
  3. README.md +181 -0
  4. SHA256SUMS +15 -0
  5. clef_text_int8_convrot_calibrated-00001-of-00014.safetensors +3 -0
  6. clef_text_int8_convrot_calibrated-00002-of-00014.safetensors +3 -0
  7. clef_text_int8_convrot_calibrated-00003-of-00014.safetensors +3 -0
  8. clef_text_int8_convrot_calibrated-00004-of-00014.safetensors +3 -0
  9. clef_text_int8_convrot_calibrated-00005-of-00014.safetensors +3 -0
  10. clef_text_int8_convrot_calibrated-00006-of-00014.safetensors +3 -0
  11. clef_text_int8_convrot_calibrated-00007-of-00014.safetensors +3 -0
  12. clef_text_int8_convrot_calibrated-00008-of-00014.safetensors +3 -0
  13. clef_text_int8_convrot_calibrated-00009-of-00014.safetensors +3 -0
  14. clef_text_int8_convrot_calibrated-00010-of-00014.safetensors +3 -0
  15. clef_text_int8_convrot_calibrated-00011-of-00014.safetensors +3 -0
  16. clef_text_int8_convrot_calibrated-00012-of-00014.safetensors +3 -0
  17. clef_text_int8_convrot_calibrated-00013-of-00014.safetensors +3 -0
  18. clef_text_int8_convrot_calibrated-00014-of-00014.safetensors +3 -0
  19. clef_text_int8_convrot_calibrated.safetensors.index.json +0 -0
  20. convrot_config.json +41 -0
  21. results/bracketed.json +1811 -0
  22. results/calibration.json +0 -0
  23. runtime/clef-bench.py +479 -0
  24. runtime/clef-bootstrap-runtime +44 -0
  25. runtime/clef-int8-bench.py +236 -0
  26. runtime/clef-int8-calibrate.py +215 -0
  27. runtime/clef-int8-calibration-check.py +56 -0
  28. runtime/clef_int8_calibration.py +275 -0
  29. runtime/clef_int8_cutlass.py +1307 -0
  30. runtime/clef_text_cute.py +341 -0
  31. runtime/clef_text_int8.py +154 -0
  32. runtime/clef_text_metrics.py +40 -0
  33. runtime/clef_text_profile.py +107 -0
  34. runtime/clef_vision_norm.py +217 -0
  35. runtime/test_clef_int8_calibration.py +97 -0
  36. runtime/test_clef_text_metrics.py +38 -0
  37. runtime/test_clef_text_profile.py +42 -0
LICENSE ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ Apache License
3
+ Version 2.0, January 2004
4
+ http://www.apache.org/licenses/
5
+
6
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
+
8
+ 1. Definitions.
9
+
10
+ "License" shall mean the terms and conditions for use, reproduction,
11
+ and distribution as defined by Sections 1 through 9 of this document.
12
+
13
+ "Licensor" shall mean the copyright owner or entity authorized by
14
+ the copyright owner that is granting the License.
15
+
16
+ "Legal Entity" shall mean the union of the acting entity and all
17
+ other entities that control, are controlled by, or are under common
18
+ control with that entity. For the purposes of this definition,
19
+ "control" means (i) the power, direct or indirect, to cause the
20
+ direction or management of such entity, whether by contract or
21
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
22
+ outstanding shares, or (iii) beneficial ownership of such entity.
23
+
24
+ "You" (or "Your") shall mean an individual or Legal Entity
25
+ exercising permissions granted by this License.
26
+
27
+ "Source" form shall mean the preferred form for making modifications,
28
+ including but not limited to software source code, documentation
29
+ source, and configuration files.
30
+
31
+ "Object" form shall mean any form resulting from mechanical
32
+ transformation or translation of a Source form, including but
33
+ not limited to compiled object code, generated documentation,
34
+ and conversions to other media types.
35
+
36
+ "Work" shall mean the work of authorship, whether in Source or
37
+ Object form, made available under the License, as indicated by a
38
+ copyright notice that is included in or attached to the work
39
+ (an example is provided in the Appendix below).
40
+
41
+ "Derivative Works" shall mean any work, whether in Source or Object
42
+ form, that is based on (or derived from) the Work and for which the
43
+ editorial revisions, annotations, elaborations, or other modifications
44
+ represent, as a whole, an original work of authorship. For the purposes
45
+ of this License, Derivative Works shall not include works that remain
46
+ separable from, or merely link (or bind by name) to the interfaces of,
47
+ the Work and Derivative Works thereof.
48
+
49
+ "Contribution" shall mean any work of authorship, including
50
+ the original version of the Work and any modifications or additions
51
+ to that Work or Derivative Works thereof, that is intentionally
52
+ submitted to Licensor for inclusion in the Work by the copyright owner
53
+ or by an individual or Legal Entity authorized to submit on behalf of
54
+ the copyright owner. For the purposes of this definition, "submitted"
55
+ means any form of electronic, verbal, or written communication sent
56
+ to the Licensor or its representatives, including but not limited to
57
+ communication on electronic mailing lists, source code control systems,
58
+ and issue tracking systems that are managed by, or on behalf of, the
59
+ Licensor for the purpose of discussing and improving the Work, but
60
+ excluding communication that is conspicuously marked or otherwise
61
+ designated in writing by the copyright owner as "Not a Contribution."
62
+
63
+ "Contributor" shall mean Licensor and any individual or Legal Entity
64
+ on behalf of whom a Contribution has been received by Licensor and
65
+ subsequently incorporated within the Work.
66
+
67
+ 2. Grant of Copyright License. Subject to the terms and conditions of
68
+ this License, each Contributor hereby grants to You a perpetual,
69
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70
+ copyright license to reproduce, prepare Derivative Works of,
71
+ publicly display, publicly perform, sublicense, and distribute the
72
+ Work and such Derivative Works in Source or Object form.
73
+
74
+ 3. Grant of Patent License. Subject to the terms and conditions of
75
+ this License, each Contributor hereby grants to You a perpetual,
76
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77
+ (except as stated in this section) patent license to make, have made,
78
+ use, offer to sell, sell, import, and otherwise transfer the Work,
79
+ where such license applies only to those patent claims licensable
80
+ by such Contributor that are necessarily infringed by their
81
+ Contribution(s) alone or by combination of their Contribution(s)
82
+ with the Work to which such Contribution(s) was submitted. If You
83
+ institute patent litigation against any entity (including a
84
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
85
+ or a Contribution incorporated within the Work constitutes direct
86
+ or contributory patent infringement, then any patent licenses
87
+ granted to You under this License for that Work shall terminate
88
+ as of the date such litigation is filed.
89
+
90
+ 4. Redistribution. You may reproduce and distribute copies of the
91
+ Work or Derivative Works thereof in any medium, with or without
92
+ modifications, and in Source or Object form, provided that You
93
+ meet the following conditions:
94
+
95
+ (a) You must give any other recipients of the Work or
96
+ Derivative Works a copy of this License; and
97
+
98
+ (b) You must cause any modified files to carry prominent notices
99
+ stating that You changed the files; and
100
+
101
+ (c) You must retain, in the Source form of any Derivative Works
102
+ that You distribute, all copyright, patent, trademark, and
103
+ attribution notices from the Source form of the Work,
104
+ excluding those notices that do not pertain to any part of
105
+ the Derivative Works; and
106
+
107
+ (d) If the Work includes a "NOTICE" text file as part of its
108
+ distribution, then any Derivative Works that You distribute must
109
+ include a readable copy of the attribution notices contained
110
+ within such NOTICE file, excluding those notices that do not
111
+ pertain to any part of the Derivative Works, in at least one
112
+ of the following places: within a NOTICE text file distributed
113
+ as part of the Derivative Works; within the Source form or
114
+ documentation, if provided along with the Derivative Works; or,
115
+ within a display generated by the Derivative Works, if and
116
+ wherever such third-party notices normally appear. The contents
117
+ of the NOTICE file are for informational purposes only and
118
+ do not modify the License. You may add Your own attribution
119
+ notices within Derivative Works that You distribute, alongside
120
+ or as an addendum to the NOTICE text from the Work, provided
121
+ that such additional attribution notices cannot be construed
122
+ as modifying the License.
123
+
124
+ You may add Your own copyright statement to Your modifications and
125
+ may provide additional or different license terms and conditions
126
+ for use, reproduction, or distribution of Your modifications, or
127
+ for any such Derivative Works as a whole, provided Your use,
128
+ reproduction, and distribution of the Work otherwise complies with
129
+ the conditions stated in this License.
130
+
131
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
132
+ any Contribution intentionally submitted for inclusion in the Work
133
+ by You to the Licensor shall be under the terms and conditions of
134
+ this License, without any additional terms or conditions.
135
+ Notwithstanding the above, nothing herein shall supersede or modify
136
+ the terms of any separate license agreement you may have executed
137
+ with Licensor regarding such Contributions.
138
+
139
+ 6. Trademarks. This License does not grant permission to use the trade
140
+ names, trademarks, service marks, or product names of the Licensor,
141
+ except as required for reasonable and customary use in describing the
142
+ origin of the Work and reproducing the content of the NOTICE file.
143
+
144
+ 7. Disclaimer of Warranty. Unless required by applicable law or
145
+ agreed to in writing, Licensor provides the Work (and each
146
+ Contributor provides its Contributions) on an "AS IS" BASIS,
147
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148
+ implied, including, without limitation, any warranties or conditions
149
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150
+ PARTICULAR PURPOSE. You are solely responsible for determining the
151
+ appropriateness of using or redistributing the Work and assume any
152
+ risks associated with Your exercise of permissions under this License.
153
+
154
+ 8. Limitation of Liability. In no event and under no legal theory,
155
+ whether in tort (including negligence), contract, or otherwise,
156
+ unless required by applicable law (such as deliberate and grossly
157
+ negligent acts) or agreed to in writing, shall any Contributor be
158
+ liable to You for damages, including any direct, indirect, special,
159
+ incidental, or consequential damages of any character arising as a
160
+ result of this License or out of the use or inability to use the
161
+ Work (including but not limited to damages for loss of goodwill,
162
+ work stoppage, computer failure or malfunction, or any and all
163
+ other commercial damages or losses), even if such Contributor
164
+ has been advised of the possibility of such damages.
165
+
166
+ 9. Accepting Warranty or Additional Liability. While redistributing
167
+ the Work or Derivative Works thereof, You may choose to offer,
168
+ and charge a fee for, acceptance of support, warranty, indemnity,
169
+ or other liability obligations and/or rights consistent with this
170
+ License. However, in accepting such obligations, You may act only
171
+ on Your own behalf and on Your sole responsibility, not on behalf
172
+ of any other Contributor, and only if You agree to indemnify,
173
+ defend, and hold each Contributor harmless for any liability
174
+ incurred by, or claims asserted against, such Contributor by reason
175
+ of your accepting any such warranty or additional liability.
176
+
177
+ END OF TERMS AND CONDITIONS
178
+
179
+ APPENDIX: How to apply the Apache License to your work.
180
+
181
+ To apply the Apache License to your work, attach the following
182
+ boilerplate notice, with the fields enclosed by brackets "[]"
183
+ replaced with your own identifying information. (Don't include
184
+ the brackets!) The text should be enclosed in the appropriate
185
+ comment syntax for the file format. We also recommend that a
186
+ file or class name and description of purpose be included on the
187
+ same "printed page" as the copyright notice for easier
188
+ identification within third-party archives.
189
+
190
+ Copyright 2026 Alibaba Cloud
191
+
192
+ Licensed under the Apache License, Version 2.0 (the "License");
193
+ you may not use this file except in compliance with the License.
194
+ You may obtain a copy of the License at
195
+
196
+ http://www.apache.org/licenses/LICENSE-2.0
197
+
198
+ Unless required by applicable law or agreed to in writing, software
199
+ distributed under the License is distributed on an "AS IS" BASIS,
200
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201
+ See the License for the specific language governing permissions and
202
+ limitations under the License.
NOTICE ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ CLEF Calibrated INT8 ConvRot Text Weights
2
+
3
+ Derived from Cloudflare/clef, revision
4
+ 2f3de3dd85f379784083b0814d997ab627200f0c, released under Apache-2.0.
5
+
6
+ Modifications: selected large text projection weights were rotated and
7
+ quantized to INT8, calibrated using per-output-row clipping and bounded
8
+ scale/bias fitting, and exported as indexed safetensors linear overrides.
9
+ Other model components are not included in this distribution.
10
+
11
+ The CuTe GEMM implementation in runtime/clef_int8_cutlass.py is adapted from
12
+ NVIDIA CUTLASS examples and retains its full BSD-3-Clause notice:
13
+ Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
14
+
15
+ Comfy Kitchen supplies the activation ConvRot quantizer as an external
16
+ dependency. Comfy Kitchen, PyTorch, Transformers, Flash Linear Attention,
17
+ CUTLASS, and other dependencies retain their respective licenses.
18
+
19
+ This derivative is not an official Cloudflare release and is not endorsed
20
+ by Cloudflare or NVIDIA.
README.md ADDED
@@ -0,0 +1,181 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ base_model: Cloudflare/clef
4
+ base_model_relation: quantized
5
+ language:
6
+ - en
7
+ tags:
8
+ - clef
9
+ - int8
10
+ - convrot
11
+ - cute-dsl
12
+ - custom-code
13
+ - experimental
14
+ ---
15
+
16
+ # CLEF Calibrated INT8 ConvRot Text Weights
17
+
18
+ Experimental calibrated W8A8 ConvRot text-linear overrides for
19
+ [Cloudflare/clef](https://huggingface.co/Cloudflare/clef/tree/2f3de3dd85f379784083b0814d997ab627200f0c).
20
+ The CuTeDSL H100 runtime applies dequantization scales and calibrated bias in
21
+ the GEMM epilogue, with no additional inference kernels.
22
+
23
+ **This is not a standalone Transformers checkpoint.** Load the original CLEF
24
+ release at revision `2f3de3dd85f379784083b0814d997ab627200f0c`, then apply these
25
+ overrides using the included runtime. `AutoModel.from_pretrained()` cannot load
26
+ this repository directly. Tokenizer, processor, embeddings, norms, small control
27
+ projections, joint decision head, and vision weights remain in the original BF16
28
+ checkpoint and are not included here.
29
+
30
+ ## Contents
31
+
32
+ - 400 calibrated large text projections: 192 MLP, 144 linear-attention, and 64
33
+ full-attention projections.
34
+ - 1,200 tensors: rotated INT8 weights, FP32 per-output-row scales, and BF16 bias.
35
+ - 14 zero-padded safetensors shards targeting 1.9 GB each; 24,350,359,552
36
+ uncompressed tensor bytes in total (22.68 GiB).
37
+ - `clef_text_int8_convrot_calibrated.safetensors.index.json`: complete tensor map,
38
+ calibration provenance, source fingerprint, and byte count.
39
+ - `SHA256SUMS`: checksums for the shards and index.
40
+ - `runtime/`: loader, CuTe kernels, calibration tools, benchmark helpers, and tests.
41
+ - `results/`: calibration and independent bracketed benchmark reports.
42
+ - `convrot_config.json`: format, source revision, runtime, and calibration settings.
43
+
44
+ The format is `clef-convrot-linear-overrides-v1`, with ConvRot group size 256.
45
+ Each projection stores `weight` shaped `[out_features, in_features]`, and
46
+ `weight_scale` and `bias` shaped `[out_features]`. Activation rotation and
47
+ row-wise INT8 quantization use the unchanged Comfy Kitchen ConvRot quantizer.
48
+
49
+ ## Calibration And Evaluation
50
+
51
+ BF16 activations were collected on the pinned real CLEF text backbone. Training
52
+ used 12 synthetic records, with 16 sampled token rows per record per projection
53
+ (192 rows). A disjoint six-record selection set provided 96 rows per projection.
54
+ Per-output-row candidates use rotated-weight absolute-max clipping fractions
55
+ `[1.0, 0.99, 0.97, 0.94, 0.90]`, a scale gain bounded to three percent around one,
56
+ and optional bias correction. The ridge coefficient is 0.05. A candidate must
57
+ reduce selection MSE by at least 0.5%; unchanged weights are retained otherwise.
58
+ This is clipping and scale/bias fitting, not GPTQ or AdaRound.
59
+
60
+ Evaluation used 16 independent synthetic held-out records plus seven older
61
+ regression probes. Agreement is with BF16 decisions, not ground-truth task
62
+ accuracy. Both INT8 variants matched 48/48 held-out decisions. Older regression
63
+ agreement increased from 30/32 to 31/32, for an overall change from 78/80 to 79/80.
64
+ One disagreement remains at 1,278 tokens. Calibration did not improve every record.
65
+
66
+ | Metric | Uncalibrated INT8 | Calibrated INT8 |
67
+ | --- | ---: | ---: |
68
+ | Mean held-out relative logit L2 | 0.02166 | 0.01837 |
69
+ | Mean held-out relative hidden-state L2 | 0.10379 | 0.09733 |
70
+ | BF16 decision agreement, all 23 records | 78/80 | 79/80 |
71
+ | Warm median inference, 1,502 tokens | 121.17 ms | 121.10 ms |
72
+ | Resident GPU memory | 28.59 GiB | 28.59 GiB |
73
+
74
+ Mean held-out relative logit drift decreased by 15.2% and hidden-state drift by
75
+ 6.2%. An independent 30-repeat benchmark measured 120.89 ms calibrated versus
76
+ 139.88 ms bracketed fused BF16 (1.157x). Shared/unshared calibrated paths matched
77
+ exactly. At 298-312 tokens, this INT8 runtime was slower than fused BF16; the
78
+ speedup is workload-dependent.
79
+
80
+ Indexed reload produced exactly identical hidden states and logits on all 23
81
+ evaluation records. All 13 unit tests passed. The calibrated GPU smoke check
82
+ also passed integer-reference epilogue parity, non-default CUDA streams, and
83
+ changed-input CUDA graph replay.
84
+
85
+ **Synthetic calibration only.** These small sets do not establish production
86
+ accuracy, probability calibration, or image/video quality. Vision weights were
87
+ not quantized, and multimodal quality of this calibrated text path was not
88
+ evaluated. Validate with representative records before deployment. This release
89
+ does not claim to reproduce the upstream Decision Index scores.
90
+
91
+ ## Runtime
92
+
93
+ Measured on NVIDIA H100 80 GB, SM90, in `nvcr.io/nvidia/pytorch:25.09-py3`:
94
+
95
+ - PyTorch `2.9.0a0+50eac811a6.nv25.09`, CUDA 13.
96
+ - Transformers `5.12.1`.
97
+ - NVIDIA CUTLASS DSL `4.8.0` and Apache TVM FFI `0.1.14.post1`.
98
+ - Comfy Kitchen `0.2.31`.
99
+ - Flash Linear Attention `0.5.2` and causal-conv1d `1.7.0`.
100
+
101
+ The CuTe backend requires inference mode, BF16 input activations, and SM90.
102
+ The measured GEMM tile is `[128, 256]` and cluster is `[2, 1]`. Recheck numerical
103
+ parity when changing hardware or library versions. A Torch integer-reference
104
+ backend is available, but the reported performance uses CuTe.
105
+
106
+ The bundled `runtime/clef-bootstrap-runtime` bootstraps dependencies in the
107
+ NVIDIA container. Set `CLEF_INSTALL_INT8=1` when invoking it; it is a Linux/CUDA
108
+ bootstrap script, not a CPU or macOS installer.
109
+
110
+ ## Loading
111
+
112
+ After installing the runtime dependencies on an H100:
113
+
114
+ ```python
115
+ import json
116
+ import sys
117
+ from pathlib import Path
118
+
119
+ import torch
120
+ from huggingface_hub import snapshot_download
121
+
122
+ source = Path(snapshot_download(
123
+ "Cloudflare/clef",
124
+ revision="2f3de3dd85f379784083b0814d997ab627200f0c",
125
+ ))
126
+ overrides = Path(snapshot_download("terminusresearch/cloudflare-clef-int8-convrot"))
127
+ sys.path.insert(0, str(source))
128
+ sys.path.insert(0, str(overrides / "runtime"))
129
+
130
+ from joint_schema_model import load_release_model, encode_record, collate_records
131
+ from clef_int8_calibration import source_digest
132
+ from clef_text_int8 import quantize_text
133
+ from clef_text_cute import patch_text
134
+
135
+ index = overrides / "clef_text_int8_convrot_calibrated.safetensors.index.json"
136
+ if json.loads(index.read_text())["metadata"]["source_digest"] != source_digest(source):
137
+ raise ValueError("Source config/code/index does not match this calibrated release")
138
+
139
+ with torch.inference_mode():
140
+ model, processor = load_release_model(source, device="cuda", dtype=torch.bfloat16)
141
+ base = (model.language_model.get_base_model()
142
+ if hasattr(model.language_model, "get_base_model") else model.language_model)
143
+ text = getattr(base.model, "language_model", base.model)
144
+ quantize_text(text, scope="all", cluster=(2, 1), calibration_index=index)
145
+ restore = patch_text(text, gva=True, gates=True, heads=True,
146
+ share_mlp_quant=True, share_attn_quant=True)
147
+
148
+ record = {
149
+ "state": {"summary": "The customer confirms service is restored."},
150
+ "questions": {
151
+ "route": {
152
+ "type": "choice",
153
+ "instructions": "Choose the appropriate handling route.",
154
+ "criteria": {"close": "Resolved issue", "review": "Unresolved issue"},
155
+ },
156
+ },
157
+ }
158
+ encoded = encode_record(processor.tokenizer, record, processor=processor)
159
+ batch = collate_records([encoded], processor.tokenizer.pad_token_id, torch.device("cuda"))
160
+ logits = model(batch)[0]
161
+ probabilities = [question_logits.float().softmax(-1).tolist() for question_logits in logits]
162
+ print(probabilities)
163
+ ```
164
+
165
+ The loader validates coverage, safe shard paths, tensor shapes/dtypes, scales,
166
+ duplicate keys, and the uncompressed byte total, and streams tensors from the
167
+ indexed shards. No weight quantization runs when loading the overrides. The
168
+ current benchmark loader first loads the original BF16 model, so startup needs
169
+ enough memory for that original model; the 28.59 GiB figure is post-conversion
170
+ residency, not the cold-load peak. This release is an inference-only overlay,
171
+ not a training checkpoint or a prepackaged Cog image.
172
+
173
+ ## License
174
+
175
+ The calibrated weights derive from the pinned Apache-2.0
176
+ [Cloudflare/clef release](https://huggingface.co/Cloudflare/clef/tree/2f3de3dd85f379784083b0814d997ab627200f0c).
177
+ `LICENSE` preserves the upstream license. `runtime/clef_int8_cutlass.py` includes
178
+ NVIDIA's BSD-3-Clause copyright notice and terms, which remain applicable to
179
+ that kernel. See `NOTICE` for modification and attribution details. This is an
180
+ experimental derivative published by Terminus Research, not an official
181
+ Cloudflare release.
SHA256SUMS ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 747660618289ecec786a7f87859c438bd9c19d7837f38c51d062540b72829bb3 clef_text_int8_convrot_calibrated-00001-of-00014.safetensors
2
+ 6bc4ad5220a3baa029a986048c14e822fd3380434df805e0e999273e49f85f35 clef_text_int8_convrot_calibrated-00002-of-00014.safetensors
3
+ c413e03e53138766b18404f60835f1f2dd386036e7557e1ffe655c568f2f6ded clef_text_int8_convrot_calibrated-00003-of-00014.safetensors
4
+ b9bd15b2bce887e9f06c42d750e661b3b1ecf4e17f645761e9d4dd77ea0d45a8 clef_text_int8_convrot_calibrated-00004-of-00014.safetensors
5
+ b24d00380e98f9b3ecede33d569c214a87183516221c0e7f7b76c19e16948972 clef_text_int8_convrot_calibrated-00005-of-00014.safetensors
6
+ 17fafd4fb52cfd432133243f81dfdb37558869d8cda06ddd9cb8405676799a80 clef_text_int8_convrot_calibrated-00006-of-00014.safetensors
7
+ ee706c7a99464f48940a31a1662eb980d8503bf1593c4116e0fccd5d2c1aa70c clef_text_int8_convrot_calibrated-00007-of-00014.safetensors
8
+ 2086b697130268f2252bf3303ceeb86639da97887b3593120f2f73592631d5b7 clef_text_int8_convrot_calibrated-00008-of-00014.safetensors
9
+ 454ec8aff5104c987d45870f2daee3f0728387cd3318235302c7fee24e5c37b2 clef_text_int8_convrot_calibrated-00009-of-00014.safetensors
10
+ 450d935a79b08ba51e84873f502be9626eb33714e46aac57f5072662ee106c90 clef_text_int8_convrot_calibrated-00010-of-00014.safetensors
11
+ 68961a270992e855dc5907e8204b2fba582b9e8f31e57003c283ddb1a2a419d7 clef_text_int8_convrot_calibrated-00011-of-00014.safetensors
12
+ ee8ab2ec02071177cd794907ed2f040951e3041db6ebe706fa1b6503441822ac clef_text_int8_convrot_calibrated-00012-of-00014.safetensors
13
+ 6a66131491b7a1ce89f6383b40c010253de0ec891c901fbb37cccfa319eebac2 clef_text_int8_convrot_calibrated-00013-of-00014.safetensors
14
+ 3fc9d98b968495ba0db5bf2a7da735edb329c7dd1ce2f455189c24c99975c52e clef_text_int8_convrot_calibrated-00014-of-00014.safetensors
15
+ c078d31e8f16c9425f34dbda14b041abe425ac66bdfe9cfa745dbf3353497494 clef_text_int8_convrot_calibrated.safetensors.index.json
clef_text_int8_convrot_calibrated-00001-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:747660618289ecec786a7f87859c438bd9c19d7837f38c51d062540b72829bb3
3
+ size 1815846088
clef_text_int8_convrot_calibrated-00002-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6bc4ad5220a3baa029a986048c14e822fd3380434df805e0e999273e49f85f35
3
+ size 1815772368
clef_text_int8_convrot_calibrated-00003-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c413e03e53138766b18404f60835f1f2dd386036e7557e1ffe655c568f2f6ded
3
+ size 1815772440
clef_text_int8_convrot_calibrated-00004-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9bd15b2bce887e9f06c42d750e661b3b1ecf4e17f645761e9d4dd77ea0d45a8
3
+ size 1894508392
clef_text_int8_convrot_calibrated-00005-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b24d00380e98f9b3ecede33d569c214a87183516221c0e7f7b76c19e16948972
3
+ size 1873511704
clef_text_int8_convrot_calibrated-00006-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:17fafd4fb52cfd432133243f81dfdb37558869d8cda06ddd9cb8405676799a80
3
+ size 1852515592
clef_text_int8_convrot_calibrated-00007-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ee706c7a99464f48940a31a1662eb980d8503bf1593c4116e0fccd5d2c1aa70c
3
+ size 1873517848
clef_text_int8_convrot_calibrated-00008-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2086b697130268f2252bf3303ceeb86639da97887b3593120f2f73592631d5b7
3
+ size 1815846176
clef_text_int8_convrot_calibrated-00009-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:454ec8aff5104c987d45870f2daee3f0728387cd3318235302c7fee24e5c37b2
3
+ size 1894508384
clef_text_int8_convrot_calibrated-00010-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:450d935a79b08ba51e84873f502be9626eb33714e46aac57f5072662ee106c90
3
+ size 1815772456
clef_text_int8_convrot_calibrated-00011-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68961a270992e855dc5907e8204b2fba582b9e8f31e57003c283ddb1a2a419d7
3
+ size 1815772456
clef_text_int8_convrot_calibrated-00012-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ee8ab2ec02071177cd794907ed2f040951e3041db6ebe706fa1b6503441822ac
3
+ size 1873511696
clef_text_int8_convrot_calibrated-00013-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6a66131491b7a1ce89f6383b40c010253de0ec891c901fbb37cccfa319eebac2
3
+ size 1894514544
clef_text_int8_convrot_calibrated-00014-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3fc9d98b968495ba0db5bf2a7da735edb329c7dd1ce2f455189c24c99975c52e
3
+ size 299115768
clef_text_int8_convrot_calibrated.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
convrot_config.json ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "clef-convrot-linear-overrides-v1",
3
+ "standalone_model": false,
4
+ "source_model": "Cloudflare/clef",
5
+ "source_revision": "2f3de3dd85f379784083b0814d997ab627200f0c",
6
+ "source_digest": "cae974bb5b4b21095342ecb5dcee32771d65a4bade3367334a762e1ae390f96e",
7
+ "scope": "all",
8
+ "group_size": 256,
9
+ "scheme": "W8A8 ConvRot",
10
+ "index": "clef_text_int8_convrot_calibrated.safetensors.index.json",
11
+ "linears": 400,
12
+ "tensors": 1200,
13
+ "total_size": 24350359552,
14
+ "calibration_data": "synthetic",
15
+ "train_records": 12,
16
+ "selection_records": 6,
17
+ "heldout_records": 16,
18
+ "sample_rows_per_record": 16,
19
+ "clips": [
20
+ 1.0,
21
+ 0.99,
22
+ 0.97,
23
+ 0.94,
24
+ 0.9
25
+ ],
26
+ "ridge": 0.05,
27
+ "max_gain": 0.03,
28
+ "min_improvement": 0.005,
29
+ "tile": [
30
+ 128,
31
+ 256
32
+ ],
33
+ "cluster": [
34
+ 2,
35
+ 1
36
+ ],
37
+ "cute_device_capability": [
38
+ 9,
39
+ 0
40
+ ]
41
+ }
results/bracketed.json ADDED
@@ -0,0 +1,1811 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_path": "Cloudflare/clef@2f3de3dd85f379784083b0814d997ab627200f0c",
3
+ "torch": "2.9.0a0+50eac811a6.nv25.09",
4
+ "device": "NVIDIA H100 80GB HBM3",
5
+ "backend": "cute",
6
+ "heads": true,
7
+ "share_mlp_input": true,
8
+ "share_attn_input": true,
9
+ "modes": {
10
+ "bf16_before": {
11
+ "cases": [
12
+ {
13
+ "case": 0,
14
+ "tokens": 1278,
15
+ "total_ms": {
16
+ "mean": 120.7420423006018,
17
+ "median": 120.32128451392055,
18
+ "p95": 126.36434938758612,
19
+ "min": 117.37496964633465,
20
+ "max": 128.86804901063442
21
+ },
22
+ "backbone_ms": {
23
+ "mean": 117.27207411701481,
24
+ "median": 117.1717680990696,
25
+ "p95": 118.6508983373642,
26
+ "min": 115.35093002021313,
27
+ "max": 118.73763613402843
28
+ },
29
+ "hidden_error": {
30
+ "max_abs": 0.0,
31
+ "relative_l2": 0.0
32
+ },
33
+ "drift": {
34
+ "logits_max_abs": 0.0,
35
+ "logits_relative_l2": 0.0,
36
+ "decisions_matched": 8,
37
+ "decisions_total": 8,
38
+ "decision_agreement": 1.0,
39
+ "reference_margin_min": 0.0625,
40
+ "reference_margin_median": 0.25,
41
+ "flips": [],
42
+ "decisions": [
43
+ {
44
+ "record": 0,
45
+ "question": 0,
46
+ "reference_choice": 0,
47
+ "candidate_choice": 0,
48
+ "matched": true,
49
+ "reference_margin": 0.3671875,
50
+ "logits_max_abs": 0.0,
51
+ "margin_exceeds_twice_error": true
52
+ },
53
+ {
54
+ "record": 0,
55
+ "question": 1,
56
+ "reference_choice": 0,
57
+ "candidate_choice": 0,
58
+ "matched": true,
59
+ "reference_margin": 0.2265625,
60
+ "logits_max_abs": 0.0,
61
+ "margin_exceeds_twice_error": true
62
+ },
63
+ {
64
+ "record": 0,
65
+ "question": 2,
66
+ "reference_choice": 0,
67
+ "candidate_choice": 0,
68
+ "matched": true,
69
+ "reference_margin": 0.2265625,
70
+ "logits_max_abs": 0.0,
71
+ "margin_exceeds_twice_error": true
72
+ },
73
+ {
74
+ "record": 0,
75
+ "question": 3,
76
+ "reference_choice": 0,
77
+ "candidate_choice": 0,
78
+ "matched": true,
79
+ "reference_margin": 0.2734375,
80
+ "logits_max_abs": 0.0,
81
+ "margin_exceeds_twice_error": true
82
+ },
83
+ {
84
+ "record": 0,
85
+ "question": 4,
86
+ "reference_choice": 0,
87
+ "candidate_choice": 0,
88
+ "matched": true,
89
+ "reference_margin": 0.419921875,
90
+ "logits_max_abs": 0.0,
91
+ "margin_exceeds_twice_error": true
92
+ },
93
+ {
94
+ "record": 0,
95
+ "question": 5,
96
+ "reference_choice": 0,
97
+ "candidate_choice": 0,
98
+ "matched": true,
99
+ "reference_margin": 0.37109375,
100
+ "logits_max_abs": 0.0,
101
+ "margin_exceeds_twice_error": true
102
+ },
103
+ {
104
+ "record": 0,
105
+ "question": 6,
106
+ "reference_choice": 0,
107
+ "candidate_choice": 0,
108
+ "matched": true,
109
+ "reference_margin": 0.0625,
110
+ "logits_max_abs": 0.0,
111
+ "margin_exceeds_twice_error": true
112
+ },
113
+ {
114
+ "record": 0,
115
+ "question": 7,
116
+ "reference_choice": 3,
117
+ "candidate_choice": 3,
118
+ "matched": true,
119
+ "reference_margin": 0.1328125,
120
+ "logits_max_abs": 0.0,
121
+ "margin_exceeds_twice_error": true
122
+ }
123
+ ]
124
+ },
125
+ "resident_vram_bytes": 55004415488,
126
+ "peak_vram_bytes": 55225683456
127
+ },
128
+ {
129
+ "case": 1,
130
+ "tokens": 1502,
131
+ "total_ms": {
132
+ "mean": 140.4689157071213,
133
+ "median": 139.46003885939717,
134
+ "p95": 146.0675811395049,
135
+ "min": 136.4539759233594,
136
+ "max": 155.13416472822428
137
+ },
138
+ "backbone_ms": {
139
+ "mean": 136.78044530873498,
140
+ "median": 136.8254255503416,
141
+ "p95": 139.53122403472662,
142
+ "min": 133.93584918230772,
143
+ "max": 140.33847115933895
144
+ },
145
+ "hidden_error": {
146
+ "max_abs": 0.0,
147
+ "relative_l2": 0.0
148
+ },
149
+ "drift": {
150
+ "logits_max_abs": 0.0,
151
+ "logits_relative_l2": 0.0,
152
+ "decisions_matched": 8,
153
+ "decisions_total": 8,
154
+ "decision_agreement": 1.0,
155
+ "reference_margin_min": 0.25,
156
+ "reference_margin_median": 0.42578125,
157
+ "flips": [],
158
+ "decisions": [
159
+ {
160
+ "record": 0,
161
+ "question": 0,
162
+ "reference_choice": 0,
163
+ "candidate_choice": 0,
164
+ "matched": true,
165
+ "reference_margin": 0.69921875,
166
+ "logits_max_abs": 0.0,
167
+ "margin_exceeds_twice_error": true
168
+ },
169
+ {
170
+ "record": 0,
171
+ "question": 1,
172
+ "reference_choice": 0,
173
+ "candidate_choice": 0,
174
+ "matched": true,
175
+ "reference_margin": 0.392578125,
176
+ "logits_max_abs": 0.0,
177
+ "margin_exceeds_twice_error": true
178
+ },
179
+ {
180
+ "record": 0,
181
+ "question": 2,
182
+ "reference_choice": 0,
183
+ "candidate_choice": 0,
184
+ "matched": true,
185
+ "reference_margin": 0.25,
186
+ "logits_max_abs": 0.0,
187
+ "margin_exceeds_twice_error": true
188
+ },
189
+ {
190
+ "record": 0,
191
+ "question": 3,
192
+ "reference_choice": 0,
193
+ "candidate_choice": 0,
194
+ "matched": true,
195
+ "reference_margin": 0.48828125,
196
+ "logits_max_abs": 0.0,
197
+ "margin_exceeds_twice_error": true
198
+ },
199
+ {
200
+ "record": 0,
201
+ "question": 4,
202
+ "reference_choice": 0,
203
+ "candidate_choice": 0,
204
+ "matched": true,
205
+ "reference_margin": 0.58984375,
206
+ "logits_max_abs": 0.0,
207
+ "margin_exceeds_twice_error": true
208
+ },
209
+ {
210
+ "record": 0,
211
+ "question": 5,
212
+ "reference_choice": 0,
213
+ "candidate_choice": 0,
214
+ "matched": true,
215
+ "reference_margin": 0.4296875,
216
+ "logits_max_abs": 0.0,
217
+ "margin_exceeds_twice_error": true
218
+ },
219
+ {
220
+ "record": 0,
221
+ "question": 6,
222
+ "reference_choice": 0,
223
+ "candidate_choice": 0,
224
+ "matched": true,
225
+ "reference_margin": 0.28515625,
226
+ "logits_max_abs": 0.0,
227
+ "margin_exceeds_twice_error": true
228
+ },
229
+ {
230
+ "record": 0,
231
+ "question": 7,
232
+ "reference_choice": 0,
233
+ "candidate_choice": 0,
234
+ "matched": true,
235
+ "reference_margin": 0.421875,
236
+ "logits_max_abs": 0.0,
237
+ "margin_exceeds_twice_error": true
238
+ }
239
+ ]
240
+ },
241
+ "resident_vram_bytes": 55004415488,
242
+ "peak_vram_bytes": 55267193344
243
+ },
244
+ {
245
+ "case": 2,
246
+ "tokens": 2270,
247
+ "total_ms": {
248
+ "mean": 207.17171784490347,
249
+ "median": 206.8792046047747,
250
+ "p95": 215.44742956757545,
251
+ "min": 200.07592719048262,
252
+ "max": 220.3005338087678
253
+ },
254
+ "backbone_ms": {
255
+ "mean": 202.72988236198822,
256
+ "median": 202.4515587836504,
257
+ "p95": 205.48917073756456,
258
+ "min": 199.79984499514103,
259
+ "max": 208.22311658412218
260
+ },
261
+ "hidden_error": {
262
+ "max_abs": 0.0,
263
+ "relative_l2": 0.0
264
+ },
265
+ "drift": {
266
+ "logits_max_abs": 0.0,
267
+ "logits_relative_l2": 0.0,
268
+ "decisions_matched": 8,
269
+ "decisions_total": 8,
270
+ "decision_agreement": 1.0,
271
+ "reference_margin_min": 0.01953125,
272
+ "reference_margin_median": 0.12109375,
273
+ "flips": [],
274
+ "decisions": [
275
+ {
276
+ "record": 0,
277
+ "question": 0,
278
+ "reference_choice": 0,
279
+ "candidate_choice": 0,
280
+ "matched": true,
281
+ "reference_margin": 0.4765625,
282
+ "logits_max_abs": 0.0,
283
+ "margin_exceeds_twice_error": true
284
+ },
285
+ {
286
+ "record": 0,
287
+ "question": 1,
288
+ "reference_choice": 0,
289
+ "candidate_choice": 0,
290
+ "matched": true,
291
+ "reference_margin": 0.306640625,
292
+ "logits_max_abs": 0.0,
293
+ "margin_exceeds_twice_error": true
294
+ },
295
+ {
296
+ "record": 0,
297
+ "question": 2,
298
+ "reference_choice": 0,
299
+ "candidate_choice": 0,
300
+ "matched": true,
301
+ "reference_margin": 0.08203125,
302
+ "logits_max_abs": 0.0,
303
+ "margin_exceeds_twice_error": true
304
+ },
305
+ {
306
+ "record": 0,
307
+ "question": 3,
308
+ "reference_choice": 0,
309
+ "candidate_choice": 0,
310
+ "matched": true,
311
+ "reference_margin": 0.1171875,
312
+ "logits_max_abs": 0.0,
313
+ "margin_exceeds_twice_error": true
314
+ },
315
+ {
316
+ "record": 0,
317
+ "question": 4,
318
+ "reference_choice": 0,
319
+ "candidate_choice": 0,
320
+ "matched": true,
321
+ "reference_margin": 0.01953125,
322
+ "logits_max_abs": 0.0,
323
+ "margin_exceeds_twice_error": true
324
+ },
325
+ {
326
+ "record": 0,
327
+ "question": 5,
328
+ "reference_choice": 0,
329
+ "candidate_choice": 0,
330
+ "matched": true,
331
+ "reference_margin": 0.05859375,
332
+ "logits_max_abs": 0.0,
333
+ "margin_exceeds_twice_error": true
334
+ },
335
+ {
336
+ "record": 0,
337
+ "question": 6,
338
+ "reference_choice": 3,
339
+ "candidate_choice": 3,
340
+ "matched": true,
341
+ "reference_margin": 0.125,
342
+ "logits_max_abs": 0.0,
343
+ "margin_exceeds_twice_error": true
344
+ },
345
+ {
346
+ "record": 0,
347
+ "question": 7,
348
+ "reference_choice": 0,
349
+ "candidate_choice": 0,
350
+ "matched": true,
351
+ "reference_margin": 0.265625,
352
+ "logits_max_abs": 0.0,
353
+ "margin_exceeds_twice_error": true
354
+ }
355
+ ]
356
+ },
357
+ "resident_vram_bytes": 55004415488,
358
+ "peak_vram_bytes": 55398181376
359
+ },
360
+ {
361
+ "case": 3,
362
+ "tokens": 298,
363
+ "total_ms": {
364
+ "mean": 66.36064605166516,
365
+ "median": 66.18425762280822,
366
+ "p95": 67.18876399099827,
367
+ "min": 65.79676549881697,
368
+ "max": 70.03371138125658
369
+ },
370
+ "backbone_ms": {
371
+ "mean": 63.170733861625195,
372
+ "median": 62.95886728912592,
373
+ "p95": 64.23373054713011,
374
+ "min": 62.67668306827545,
375
+ "max": 64.84911683946848
376
+ },
377
+ "hidden_error": {
378
+ "max_abs": 0.0,
379
+ "relative_l2": 0.0
380
+ },
381
+ "drift": {
382
+ "logits_max_abs": 0.0,
383
+ "logits_relative_l2": 0.0,
384
+ "decisions_matched": 2,
385
+ "decisions_total": 2,
386
+ "decision_agreement": 1.0,
387
+ "reference_margin_min": 4.31640625,
388
+ "reference_margin_median": 4.353515625,
389
+ "flips": [],
390
+ "decisions": [
391
+ {
392
+ "record": 0,
393
+ "question": 0,
394
+ "reference_choice": 3,
395
+ "candidate_choice": 3,
396
+ "matched": true,
397
+ "reference_margin": 4.390625,
398
+ "logits_max_abs": 0.0,
399
+ "margin_exceeds_twice_error": true
400
+ },
401
+ {
402
+ "record": 0,
403
+ "question": 1,
404
+ "reference_choice": 1,
405
+ "candidate_choice": 1,
406
+ "matched": true,
407
+ "reference_margin": 4.31640625,
408
+ "logits_max_abs": 0.0,
409
+ "margin_exceeds_twice_error": true
410
+ }
411
+ ]
412
+ },
413
+ "resident_vram_bytes": 55004415488,
414
+ "peak_vram_bytes": 55057560576
415
+ },
416
+ {
417
+ "case": 4,
418
+ "tokens": 303,
419
+ "total_ms": {
420
+ "mean": 66.08876610795657,
421
+ "median": 66.01279973983765,
422
+ "p95": 66.44898280501366,
423
+ "min": 65.63741154968739,
424
+ "max": 67.82077439129353
425
+ },
426
+ "backbone_ms": {
427
+ "mean": 62.835137452930205,
428
+ "median": 62.80959444120526,
429
+ "p95": 63.14793787896633,
430
+ "min": 62.608824111521244,
431
+ "max": 63.26063070446253
432
+ },
433
+ "hidden_error": {
434
+ "max_abs": 0.0,
435
+ "relative_l2": 0.0
436
+ },
437
+ "drift": {
438
+ "logits_max_abs": 0.0,
439
+ "logits_relative_l2": 0.0,
440
+ "decisions_matched": 2,
441
+ "decisions_total": 2,
442
+ "decision_agreement": 1.0,
443
+ "reference_margin_min": 2.8984375,
444
+ "reference_margin_median": 3.73046875,
445
+ "flips": [],
446
+ "decisions": [
447
+ {
448
+ "record": 0,
449
+ "question": 0,
450
+ "reference_choice": 2,
451
+ "candidate_choice": 2,
452
+ "matched": true,
453
+ "reference_margin": 4.5625,
454
+ "logits_max_abs": 0.0,
455
+ "margin_exceeds_twice_error": true
456
+ },
457
+ {
458
+ "record": 0,
459
+ "question": 1,
460
+ "reference_choice": 0,
461
+ "candidate_choice": 0,
462
+ "matched": true,
463
+ "reference_margin": 2.8984375,
464
+ "logits_max_abs": 0.0,
465
+ "margin_exceeds_twice_error": true
466
+ }
467
+ ]
468
+ },
469
+ "resident_vram_bytes": 55004415488,
470
+ "peak_vram_bytes": 55058505216
471
+ },
472
+ {
473
+ "case": 5,
474
+ "tokens": 304,
475
+ "total_ms": {
476
+ "mean": 66.5853524270157,
477
+ "median": 66.38480117544532,
478
+ "p95": 68.45066975802183,
479
+ "min": 65.95136970281601,
480
+ "max": 69.19064838439226
481
+ },
482
+ "backbone_ms": {
483
+ "mean": 63.43956707666318,
484
+ "median": 63.35389753803611,
485
+ "p95": 64.41042199730873,
486
+ "min": 62.95953691005707,
487
+ "max": 65.36273565143347
488
+ },
489
+ "hidden_error": {
490
+ "max_abs": 0.0,
491
+ "relative_l2": 0.0
492
+ },
493
+ "drift": {
494
+ "logits_max_abs": 0.0,
495
+ "logits_relative_l2": 0.0,
496
+ "decisions_matched": 2,
497
+ "decisions_total": 2,
498
+ "decision_agreement": 1.0,
499
+ "reference_margin_min": 4.2109375,
500
+ "reference_margin_median": 4.4228515625,
501
+ "flips": [],
502
+ "decisions": [
503
+ {
504
+ "record": 0,
505
+ "question": 0,
506
+ "reference_choice": 0,
507
+ "candidate_choice": 0,
508
+ "matched": true,
509
+ "reference_margin": 4.2109375,
510
+ "logits_max_abs": 0.0,
511
+ "margin_exceeds_twice_error": true
512
+ },
513
+ {
514
+ "record": 0,
515
+ "question": 1,
516
+ "reference_choice": 1,
517
+ "candidate_choice": 1,
518
+ "matched": true,
519
+ "reference_margin": 4.634765625,
520
+ "logits_max_abs": 0.0,
521
+ "margin_exceeds_twice_error": true
522
+ }
523
+ ]
524
+ },
525
+ "resident_vram_bytes": 55004415488,
526
+ "peak_vram_bytes": 55058544128
527
+ },
528
+ {
529
+ "case": 6,
530
+ "tokens": 312,
531
+ "total_ms": {
532
+ "mean": 66.48833425715566,
533
+ "median": 66.47943751886487,
534
+ "p95": 66.86687096953392,
535
+ "min": 66.1541111767292,
536
+ "max": 66.8799402192235
537
+ },
538
+ "backbone_ms": {
539
+ "mean": 63.53455794354281,
540
+ "median": 63.32046305760741,
541
+ "p95": 63.97620867937803,
542
+ "min": 63.109404407441616,
543
+ "max": 68.88721697032452
544
+ },
545
+ "hidden_error": {
546
+ "max_abs": 0.0,
547
+ "relative_l2": 0.0
548
+ },
549
+ "drift": {
550
+ "logits_max_abs": 0.0,
551
+ "logits_relative_l2": 0.0,
552
+ "decisions_matched": 2,
553
+ "decisions_total": 2,
554
+ "decision_agreement": 1.0,
555
+ "reference_margin_min": 1.7109375,
556
+ "reference_margin_median": 2.55859375,
557
+ "flips": [],
558
+ "decisions": [
559
+ {
560
+ "record": 0,
561
+ "question": 0,
562
+ "reference_choice": 2,
563
+ "candidate_choice": 2,
564
+ "matched": true,
565
+ "reference_margin": 3.40625,
566
+ "logits_max_abs": 0.0,
567
+ "margin_exceeds_twice_error": true
568
+ },
569
+ {
570
+ "record": 0,
571
+ "question": 1,
572
+ "reference_choice": 0,
573
+ "candidate_choice": 0,
574
+ "matched": true,
575
+ "reference_margin": 1.7109375,
576
+ "logits_max_abs": 0.0,
577
+ "margin_exceeds_twice_error": true
578
+ }
579
+ ]
580
+ },
581
+ "resident_vram_bytes": 55004415488,
582
+ "peak_vram_bytes": 55060240896
583
+ }
584
+ ],
585
+ "conversion": null,
586
+ "shared_input_exact": null,
587
+ "patch_statistics": {
588
+ "rms_calls": 43456,
589
+ "swiglu_calls": 28672,
590
+ "residual_calls": 28672,
591
+ "delta_calls": 21504,
592
+ "shared_quant_calls": 0,
593
+ "shared_attn_quant_calls": 0,
594
+ "fallbacks": 0
595
+ }
596
+ },
597
+ "all": {
598
+ "cases": [
599
+ {
600
+ "case": 0,
601
+ "tokens": 1278,
602
+ "total_ms": {
603
+ "mean": 107.05213258042932,
604
+ "median": 107.02321119606495,
605
+ "p95": 107.38053172826767,
606
+ "min": 106.93768132477999,
607
+ "max": 107.4241129681468
608
+ },
609
+ "backbone_ms": {
610
+ "mean": 101.29548273980618,
611
+ "median": 101.28400847315788,
612
+ "p95": 101.40760242938995,
613
+ "min": 101.21874045580626,
614
+ "max": 101.45805310457945
615
+ },
616
+ "hidden_error": {
617
+ "max_abs": 30.8125,
618
+ "relative_l2": 0.13932070136070251
619
+ },
620
+ "drift": {
621
+ "logits_max_abs": 0.234375,
622
+ "logits_relative_l2": 0.24432059284457605,
623
+ "decisions_matched": 7,
624
+ "decisions_total": 8,
625
+ "decision_agreement": 0.875,
626
+ "reference_margin_min": 0.0625,
627
+ "reference_margin_median": 0.25,
628
+ "flips": [
629
+ {
630
+ "record": 0,
631
+ "question": 7,
632
+ "reference_choice": 3,
633
+ "candidate_choice": 0,
634
+ "matched": false,
635
+ "reference_margin": 0.1328125,
636
+ "logits_max_abs": 0.234375,
637
+ "margin_exceeds_twice_error": false
638
+ }
639
+ ],
640
+ "decisions": [
641
+ {
642
+ "record": 0,
643
+ "question": 0,
644
+ "reference_choice": 0,
645
+ "candidate_choice": 0,
646
+ "matched": true,
647
+ "reference_margin": 0.3671875,
648
+ "logits_max_abs": 0.15625,
649
+ "margin_exceeds_twice_error": true
650
+ },
651
+ {
652
+ "record": 0,
653
+ "question": 1,
654
+ "reference_choice": 0,
655
+ "candidate_choice": 0,
656
+ "matched": true,
657
+ "reference_margin": 0.2265625,
658
+ "logits_max_abs": 0.109375,
659
+ "margin_exceeds_twice_error": true
660
+ },
661
+ {
662
+ "record": 0,
663
+ "question": 2,
664
+ "reference_choice": 0,
665
+ "candidate_choice": 0,
666
+ "matched": true,
667
+ "reference_margin": 0.2265625,
668
+ "logits_max_abs": 0.080078125,
669
+ "margin_exceeds_twice_error": true
670
+ },
671
+ {
672
+ "record": 0,
673
+ "question": 3,
674
+ "reference_choice": 0,
675
+ "candidate_choice": 0,
676
+ "matched": true,
677
+ "reference_margin": 0.2734375,
678
+ "logits_max_abs": 0.07421875,
679
+ "margin_exceeds_twice_error": true
680
+ },
681
+ {
682
+ "record": 0,
683
+ "question": 4,
684
+ "reference_choice": 0,
685
+ "candidate_choice": 0,
686
+ "matched": true,
687
+ "reference_margin": 0.419921875,
688
+ "logits_max_abs": 0.119140625,
689
+ "margin_exceeds_twice_error": true
690
+ },
691
+ {
692
+ "record": 0,
693
+ "question": 5,
694
+ "reference_choice": 0,
695
+ "candidate_choice": 0,
696
+ "matched": true,
697
+ "reference_margin": 0.37109375,
698
+ "logits_max_abs": 0.109375,
699
+ "margin_exceeds_twice_error": true
700
+ },
701
+ {
702
+ "record": 0,
703
+ "question": 6,
704
+ "reference_choice": 0,
705
+ "candidate_choice": 0,
706
+ "matched": true,
707
+ "reference_margin": 0.0625,
708
+ "logits_max_abs": 0.099609375,
709
+ "margin_exceeds_twice_error": false
710
+ },
711
+ {
712
+ "record": 0,
713
+ "question": 7,
714
+ "reference_choice": 3,
715
+ "candidate_choice": 0,
716
+ "matched": false,
717
+ "reference_margin": 0.1328125,
718
+ "logits_max_abs": 0.234375,
719
+ "margin_exceeds_twice_error": false
720
+ }
721
+ ]
722
+ },
723
+ "resident_vram_bytes": 30701897216,
724
+ "peak_vram_bytes": 30936492544,
725
+ "speedup_vs_fused_bf16": 1.125855436584257
726
+ },
727
+ {
728
+ "case": 1,
729
+ "tokens": 1502,
730
+ "total_ms": {
731
+ "mean": 120.8846031067272,
732
+ "median": 120.88794028386474,
733
+ "p95": 121.01801671087742,
734
+ "min": 120.7155492156744,
735
+ "max": 121.02974485605955
736
+ },
737
+ "backbone_ms": {
738
+ "mean": 115.24689175809424,
739
+ "median": 115.23000663146377,
740
+ "p95": 115.4045844450593,
741
+ "min": 115.11011142283678,
742
+ "max": 115.42476527392864
743
+ },
744
+ "hidden_error": {
745
+ "max_abs": 31.283203125,
746
+ "relative_l2": 0.2085031121969223
747
+ },
748
+ "drift": {
749
+ "logits_max_abs": 0.29296875,
750
+ "logits_relative_l2": 0.5195755740972049,
751
+ "decisions_matched": 8,
752
+ "decisions_total": 8,
753
+ "decision_agreement": 1.0,
754
+ "reference_margin_min": 0.25,
755
+ "reference_margin_median": 0.42578125,
756
+ "flips": [],
757
+ "decisions": [
758
+ {
759
+ "record": 0,
760
+ "question": 0,
761
+ "reference_choice": 0,
762
+ "candidate_choice": 0,
763
+ "matched": true,
764
+ "reference_margin": 0.69921875,
765
+ "logits_max_abs": 0.29296875,
766
+ "margin_exceeds_twice_error": true
767
+ },
768
+ {
769
+ "record": 0,
770
+ "question": 1,
771
+ "reference_choice": 0,
772
+ "candidate_choice": 0,
773
+ "matched": true,
774
+ "reference_margin": 0.392578125,
775
+ "logits_max_abs": 0.267578125,
776
+ "margin_exceeds_twice_error": false
777
+ },
778
+ {
779
+ "record": 0,
780
+ "question": 2,
781
+ "reference_choice": 0,
782
+ "candidate_choice": 0,
783
+ "matched": true,
784
+ "reference_margin": 0.25,
785
+ "logits_max_abs": 0.26171875,
786
+ "margin_exceeds_twice_error": false
787
+ },
788
+ {
789
+ "record": 0,
790
+ "question": 3,
791
+ "reference_choice": 0,
792
+ "candidate_choice": 0,
793
+ "matched": true,
794
+ "reference_margin": 0.48828125,
795
+ "logits_max_abs": 0.24609375,
796
+ "margin_exceeds_twice_error": false
797
+ },
798
+ {
799
+ "record": 0,
800
+ "question": 4,
801
+ "reference_choice": 0,
802
+ "candidate_choice": 0,
803
+ "matched": true,
804
+ "reference_margin": 0.58984375,
805
+ "logits_max_abs": 0.2734375,
806
+ "margin_exceeds_twice_error": true
807
+ },
808
+ {
809
+ "record": 0,
810
+ "question": 5,
811
+ "reference_choice": 0,
812
+ "candidate_choice": 0,
813
+ "matched": true,
814
+ "reference_margin": 0.4296875,
815
+ "logits_max_abs": 0.23828125,
816
+ "margin_exceeds_twice_error": false
817
+ },
818
+ {
819
+ "record": 0,
820
+ "question": 6,
821
+ "reference_choice": 0,
822
+ "candidate_choice": 0,
823
+ "matched": true,
824
+ "reference_margin": 0.28515625,
825
+ "logits_max_abs": 0.28125,
826
+ "margin_exceeds_twice_error": false
827
+ },
828
+ {
829
+ "record": 0,
830
+ "question": 7,
831
+ "reference_choice": 0,
832
+ "candidate_choice": 0,
833
+ "matched": true,
834
+ "reference_margin": 0.421875,
835
+ "logits_max_abs": 0.22265625,
836
+ "margin_exceeds_twice_error": false
837
+ }
838
+ ]
839
+ },
840
+ "resident_vram_bytes": 30701897216,
841
+ "peak_vram_bytes": 30979767808,
842
+ "speedup_vs_fused_bf16": 1.1570775125081916
843
+ },
844
+ {
845
+ "case": 2,
846
+ "tokens": 2270,
847
+ "total_ms": {
848
+ "mean": 172.77655890211463,
849
+ "median": 172.73904010653496,
850
+ "p95": 172.9419371113181,
851
+ "min": 172.47775755822659,
852
+ "max": 174.69287477433681
853
+ },
854
+ "backbone_ms": {
855
+ "mean": 166.9653277223309,
856
+ "median": 166.96942318230867,
857
+ "p95": 167.18766652047634,
858
+ "min": 166.6247695684433,
859
+ "max": 167.29615069925785
860
+ },
861
+ "hidden_error": {
862
+ "max_abs": 45.296875,
863
+ "relative_l2": 0.2817922830581665
864
+ },
865
+ "drift": {
866
+ "logits_max_abs": 0.21484375,
867
+ "logits_relative_l2": 0.3134054938237255,
868
+ "decisions_matched": 8,
869
+ "decisions_total": 8,
870
+ "decision_agreement": 1.0,
871
+ "reference_margin_min": 0.01953125,
872
+ "reference_margin_median": 0.12109375,
873
+ "flips": [],
874
+ "decisions": [
875
+ {
876
+ "record": 0,
877
+ "question": 0,
878
+ "reference_choice": 0,
879
+ "candidate_choice": 0,
880
+ "matched": true,
881
+ "reference_margin": 0.4765625,
882
+ "logits_max_abs": 0.1484375,
883
+ "margin_exceeds_twice_error": true
884
+ },
885
+ {
886
+ "record": 0,
887
+ "question": 1,
888
+ "reference_choice": 0,
889
+ "candidate_choice": 0,
890
+ "matched": true,
891
+ "reference_margin": 0.306640625,
892
+ "logits_max_abs": 0.111328125,
893
+ "margin_exceeds_twice_error": true
894
+ },
895
+ {
896
+ "record": 0,
897
+ "question": 2,
898
+ "reference_choice": 0,
899
+ "candidate_choice": 0,
900
+ "matched": true,
901
+ "reference_margin": 0.08203125,
902
+ "logits_max_abs": 0.1484375,
903
+ "margin_exceeds_twice_error": false
904
+ },
905
+ {
906
+ "record": 0,
907
+ "question": 3,
908
+ "reference_choice": 0,
909
+ "candidate_choice": 0,
910
+ "matched": true,
911
+ "reference_margin": 0.1171875,
912
+ "logits_max_abs": 0.1171875,
913
+ "margin_exceeds_twice_error": false
914
+ },
915
+ {
916
+ "record": 0,
917
+ "question": 4,
918
+ "reference_choice": 0,
919
+ "candidate_choice": 0,
920
+ "matched": true,
921
+ "reference_margin": 0.01953125,
922
+ "logits_max_abs": 0.13671875,
923
+ "margin_exceeds_twice_error": false
924
+ },
925
+ {
926
+ "record": 0,
927
+ "question": 5,
928
+ "reference_choice": 0,
929
+ "candidate_choice": 0,
930
+ "matched": true,
931
+ "reference_margin": 0.05859375,
932
+ "logits_max_abs": 0.08984375,
933
+ "margin_exceeds_twice_error": false
934
+ },
935
+ {
936
+ "record": 0,
937
+ "question": 6,
938
+ "reference_choice": 3,
939
+ "candidate_choice": 3,
940
+ "matched": true,
941
+ "reference_margin": 0.125,
942
+ "logits_max_abs": 0.15625,
943
+ "margin_exceeds_twice_error": false
944
+ },
945
+ {
946
+ "record": 0,
947
+ "question": 7,
948
+ "reference_choice": 0,
949
+ "candidate_choice": 0,
950
+ "matched": true,
951
+ "reference_margin": 0.265625,
952
+ "logits_max_abs": 0.21484375,
953
+ "margin_exceeds_twice_error": false
954
+ }
955
+ ]
956
+ },
957
+ "resident_vram_bytes": 30701897216,
958
+ "peak_vram_bytes": 31118588416,
959
+ "speedup_vs_fused_bf16": 1.1976097894375464
960
+ },
961
+ {
962
+ "case": 3,
963
+ "tokens": 298,
964
+ "total_ms": {
965
+ "mean": 82.499930759271,
966
+ "median": 82.49658718705177,
967
+ "p95": 82.73806516081095,
968
+ "min": 82.13112596422434,
969
+ "max": 82.77752809226513
970
+ },
971
+ "backbone_ms": {
972
+ "mean": 79.38881159449616,
973
+ "median": 79.29326640442014,
974
+ "p95": 80.56880813091993,
975
+ "min": 78.97979486733675,
976
+ "max": 80.72016015648842
977
+ },
978
+ "hidden_error": {
979
+ "max_abs": 8.25,
980
+ "relative_l2": 0.08705292642116547
981
+ },
982
+ "drift": {
983
+ "logits_max_abs": 0.171875,
984
+ "logits_relative_l2": 0.027874615994997076,
985
+ "decisions_matched": 2,
986
+ "decisions_total": 2,
987
+ "decision_agreement": 1.0,
988
+ "reference_margin_min": 4.31640625,
989
+ "reference_margin_median": 4.353515625,
990
+ "flips": [],
991
+ "decisions": [
992
+ {
993
+ "record": 0,
994
+ "question": 0,
995
+ "reference_choice": 3,
996
+ "candidate_choice": 3,
997
+ "matched": true,
998
+ "reference_margin": 4.390625,
999
+ "logits_max_abs": 0.0625,
1000
+ "margin_exceeds_twice_error": true
1001
+ },
1002
+ {
1003
+ "record": 0,
1004
+ "question": 1,
1005
+ "reference_choice": 1,
1006
+ "candidate_choice": 1,
1007
+ "matched": true,
1008
+ "reference_margin": 4.31640625,
1009
+ "logits_max_abs": 0.171875,
1010
+ "margin_exceeds_twice_error": true
1011
+ }
1012
+ ]
1013
+ },
1014
+ "resident_vram_bytes": 30701897216,
1015
+ "peak_vram_bytes": 30758282240,
1016
+ "speedup_vs_fused_bf16": 0.7940032277701942
1017
+ },
1018
+ {
1019
+ "case": 4,
1020
+ "tokens": 303,
1021
+ "total_ms": {
1022
+ "mean": 83.24504966537158,
1023
+ "median": 82.82628236338496,
1024
+ "p95": 87.72443886846304,
1025
+ "min": 82.26131554692984,
1026
+ "max": 87.76593301445246
1027
+ },
1028
+ "backbone_ms": {
1029
+ "mean": 79.43275828535359,
1030
+ "median": 79.20759683474898,
1031
+ "p95": 79.86426446586847,
1032
+ "min": 78.9574459195137,
1033
+ "max": 83.8590394705534
1034
+ },
1035
+ "hidden_error": {
1036
+ "max_abs": 10.75,
1037
+ "relative_l2": 0.0930745080113411
1038
+ },
1039
+ "drift": {
1040
+ "logits_max_abs": 0.09375,
1041
+ "logits_relative_l2": 0.012005244385097194,
1042
+ "decisions_matched": 2,
1043
+ "decisions_total": 2,
1044
+ "decision_agreement": 1.0,
1045
+ "reference_margin_min": 2.8984375,
1046
+ "reference_margin_median": 3.73046875,
1047
+ "flips": [],
1048
+ "decisions": [
1049
+ {
1050
+ "record": 0,
1051
+ "question": 0,
1052
+ "reference_choice": 2,
1053
+ "candidate_choice": 2,
1054
+ "matched": true,
1055
+ "reference_margin": 4.5625,
1056
+ "logits_max_abs": 0.09375,
1057
+ "margin_exceeds_twice_error": true
1058
+ },
1059
+ {
1060
+ "record": 0,
1061
+ "question": 1,
1062
+ "reference_choice": 0,
1063
+ "candidate_choice": 0,
1064
+ "matched": true,
1065
+ "reference_margin": 2.8984375,
1066
+ "logits_max_abs": 0.0078125,
1067
+ "margin_exceeds_twice_error": true
1068
+ }
1069
+ ]
1070
+ },
1071
+ "resident_vram_bytes": 30701897216,
1072
+ "peak_vram_bytes": 30759723520,
1073
+ "speedup_vs_fused_bf16": 0.7882684823613131
1074
+ },
1075
+ {
1076
+ "case": 5,
1077
+ "tokens": 304,
1078
+ "total_ms": {
1079
+ "mean": 82.33912341917554,
1080
+ "median": 82.14510511606932,
1081
+ "p95": 84.59722064435482,
1082
+ "min": 81.95048198103905,
1083
+ "max": 85.46646125614643
1084
+ },
1085
+ "backbone_ms": {
1086
+ "mean": 79.76332195103168,
1087
+ "median": 79.51081171631813,
1088
+ "p95": 80.94685152173042,
1089
+ "min": 79.24429979175329,
1090
+ "max": 81.1643386259675
1091
+ },
1092
+ "hidden_error": {
1093
+ "max_abs": 15.8125,
1094
+ "relative_l2": 0.06573484092950821
1095
+ },
1096
+ "drift": {
1097
+ "logits_max_abs": 0.03125,
1098
+ "logits_relative_l2": 0.007555761286739542,
1099
+ "decisions_matched": 2,
1100
+ "decisions_total": 2,
1101
+ "decision_agreement": 1.0,
1102
+ "reference_margin_min": 4.2109375,
1103
+ "reference_margin_median": 4.4228515625,
1104
+ "flips": [],
1105
+ "decisions": [
1106
+ {
1107
+ "record": 0,
1108
+ "question": 0,
1109
+ "reference_choice": 0,
1110
+ "candidate_choice": 0,
1111
+ "matched": true,
1112
+ "reference_margin": 4.2109375,
1113
+ "logits_max_abs": 0.03125,
1114
+ "margin_exceeds_twice_error": true
1115
+ },
1116
+ {
1117
+ "record": 0,
1118
+ "question": 1,
1119
+ "reference_choice": 1,
1120
+ "candidate_choice": 1,
1121
+ "matched": true,
1122
+ "reference_margin": 4.634765625,
1123
+ "logits_max_abs": 0.03125,
1124
+ "margin_exceeds_twice_error": true
1125
+ }
1126
+ ]
1127
+ },
1128
+ "resident_vram_bytes": 30701897216,
1129
+ "peak_vram_bytes": 30759831040,
1130
+ "speedup_vs_fused_bf16": 0.796956165198466
1131
+ },
1132
+ {
1133
+ "case": 6,
1134
+ "tokens": 312,
1135
+ "total_ms": {
1136
+ "mean": 81.95691602304578,
1137
+ "median": 81.7097662948072,
1138
+ "p95": 82.6884787529707,
1139
+ "min": 81.4859289675951,
1140
+ "max": 87.34696544706821
1141
+ },
1142
+ "backbone_ms": {
1143
+ "mean": 79.5031982474029,
1144
+ "median": 78.90532445162535,
1145
+ "p95": 82.72027503699064,
1146
+ "min": 78.47033347934484,
1147
+ "max": 82.83683657646179
1148
+ },
1149
+ "hidden_error": {
1150
+ "max_abs": 11.75390625,
1151
+ "relative_l2": 0.09043972939252853
1152
+ },
1153
+ "drift": {
1154
+ "logits_max_abs": 0.0625,
1155
+ "logits_relative_l2": 0.01652767290549088,
1156
+ "decisions_matched": 2,
1157
+ "decisions_total": 2,
1158
+ "decision_agreement": 1.0,
1159
+ "reference_margin_min": 1.7109375,
1160
+ "reference_margin_median": 2.55859375,
1161
+ "flips": [],
1162
+ "decisions": [
1163
+ {
1164
+ "record": 0,
1165
+ "question": 0,
1166
+ "reference_choice": 2,
1167
+ "candidate_choice": 2,
1168
+ "matched": true,
1169
+ "reference_margin": 3.40625,
1170
+ "logits_max_abs": 0.0625,
1171
+ "margin_exceeds_twice_error": true
1172
+ },
1173
+ {
1174
+ "record": 0,
1175
+ "question": 1,
1176
+ "reference_choice": 0,
1177
+ "candidate_choice": 0,
1178
+ "matched": true,
1179
+ "reference_margin": 1.7109375,
1180
+ "logits_max_abs": 0.0625,
1181
+ "margin_exceeds_twice_error": true
1182
+ }
1183
+ ]
1184
+ },
1185
+ "resident_vram_bytes": 30701897216,
1186
+ "peak_vram_bytes": 30762929664,
1187
+ "speedup_vs_fused_bf16": 0.8054773965934207
1188
+ }
1189
+ ],
1190
+ "conversion": {
1191
+ "scope": "all",
1192
+ "linears": 400,
1193
+ "group_size": 256,
1194
+ "original_weight_bytes": 48653926400,
1195
+ "quantized_weight_bytes": 24350359552,
1196
+ "backend": "cute",
1197
+ "tile": [
1198
+ 128,
1199
+ 256
1200
+ ],
1201
+ "cluster": [
1202
+ 2,
1203
+ 1
1204
+ ],
1205
+ "calibration_index": "clef_text_int8_convrot_calibrated.safetensors.index.json",
1206
+ "total_quantized_linears": 400,
1207
+ "seconds": 3.488601409830153
1208
+ },
1209
+ "shared_input_exact": true,
1210
+ "patch_statistics": {
1211
+ "rms_calls": 43456,
1212
+ "swiglu_calls": 28672,
1213
+ "residual_calls": 28672,
1214
+ "delta_calls": 21504,
1215
+ "shared_quant_calls": 28672,
1216
+ "shared_attn_quant_calls": 21504,
1217
+ "fallbacks": 0
1218
+ }
1219
+ },
1220
+ "bf16_after": {
1221
+ "cases": [
1222
+ {
1223
+ "case": 0,
1224
+ "tokens": 1278,
1225
+ "total_ms": {
1226
+ "mean": 121.42789217953882,
1227
+ "median": 120.66404381766915,
1228
+ "p95": 127.8464999049902,
1229
+ "min": 116.93222634494305,
1230
+ "max": 133.81348084658384
1231
+ },
1232
+ "backbone_ms": {
1233
+ "mean": 117.61606192837158,
1234
+ "median": 117.35368939116597,
1235
+ "p95": 119.93260867893696,
1236
+ "min": 115.65397959202528,
1237
+ "max": 120.16370613127947
1238
+ },
1239
+ "hidden_error": {
1240
+ "max_abs": 0.0,
1241
+ "relative_l2": 0.0
1242
+ },
1243
+ "drift": {
1244
+ "logits_max_abs": 0.0,
1245
+ "logits_relative_l2": 0.0,
1246
+ "decisions_matched": 8,
1247
+ "decisions_total": 8,
1248
+ "decision_agreement": 1.0,
1249
+ "reference_margin_min": 0.0625,
1250
+ "reference_margin_median": 0.25,
1251
+ "flips": [],
1252
+ "decisions": [
1253
+ {
1254
+ "record": 0,
1255
+ "question": 0,
1256
+ "reference_choice": 0,
1257
+ "candidate_choice": 0,
1258
+ "matched": true,
1259
+ "reference_margin": 0.3671875,
1260
+ "logits_max_abs": 0.0,
1261
+ "margin_exceeds_twice_error": true
1262
+ },
1263
+ {
1264
+ "record": 0,
1265
+ "question": 1,
1266
+ "reference_choice": 0,
1267
+ "candidate_choice": 0,
1268
+ "matched": true,
1269
+ "reference_margin": 0.2265625,
1270
+ "logits_max_abs": 0.0,
1271
+ "margin_exceeds_twice_error": true
1272
+ },
1273
+ {
1274
+ "record": 0,
1275
+ "question": 2,
1276
+ "reference_choice": 0,
1277
+ "candidate_choice": 0,
1278
+ "matched": true,
1279
+ "reference_margin": 0.2265625,
1280
+ "logits_max_abs": 0.0,
1281
+ "margin_exceeds_twice_error": true
1282
+ },
1283
+ {
1284
+ "record": 0,
1285
+ "question": 3,
1286
+ "reference_choice": 0,
1287
+ "candidate_choice": 0,
1288
+ "matched": true,
1289
+ "reference_margin": 0.2734375,
1290
+ "logits_max_abs": 0.0,
1291
+ "margin_exceeds_twice_error": true
1292
+ },
1293
+ {
1294
+ "record": 0,
1295
+ "question": 4,
1296
+ "reference_choice": 0,
1297
+ "candidate_choice": 0,
1298
+ "matched": true,
1299
+ "reference_margin": 0.419921875,
1300
+ "logits_max_abs": 0.0,
1301
+ "margin_exceeds_twice_error": true
1302
+ },
1303
+ {
1304
+ "record": 0,
1305
+ "question": 5,
1306
+ "reference_choice": 0,
1307
+ "candidate_choice": 0,
1308
+ "matched": true,
1309
+ "reference_margin": 0.37109375,
1310
+ "logits_max_abs": 0.0,
1311
+ "margin_exceeds_twice_error": true
1312
+ },
1313
+ {
1314
+ "record": 0,
1315
+ "question": 6,
1316
+ "reference_choice": 0,
1317
+ "candidate_choice": 0,
1318
+ "matched": true,
1319
+ "reference_margin": 0.0625,
1320
+ "logits_max_abs": 0.0,
1321
+ "margin_exceeds_twice_error": true
1322
+ },
1323
+ {
1324
+ "record": 0,
1325
+ "question": 7,
1326
+ "reference_choice": 3,
1327
+ "candidate_choice": 3,
1328
+ "matched": true,
1329
+ "reference_margin": 0.1328125,
1330
+ "logits_max_abs": 0.0,
1331
+ "margin_exceeds_twice_error": true
1332
+ }
1333
+ ]
1334
+ },
1335
+ "resident_vram_bytes": 55004415488,
1336
+ "peak_vram_bytes": 55225683456
1337
+ },
1338
+ {
1339
+ "case": 1,
1340
+ "tokens": 1502,
1341
+ "total_ms": {
1342
+ "mean": 140.32531641423702,
1343
+ "median": 140.29339561238885,
1344
+ "p95": 142.69945956766605,
1345
+ "min": 137.4212009832263,
1346
+ "max": 143.30243784934282
1347
+ },
1348
+ "backbone_ms": {
1349
+ "mean": 136.75211233397326,
1350
+ "median": 136.8663413450122,
1351
+ "p95": 138.3480206131935,
1352
+ "min": 132.71950464695692,
1353
+ "max": 139.0366218984127
1354
+ },
1355
+ "hidden_error": {
1356
+ "max_abs": 0.0,
1357
+ "relative_l2": 0.0
1358
+ },
1359
+ "drift": {
1360
+ "logits_max_abs": 0.0,
1361
+ "logits_relative_l2": 0.0,
1362
+ "decisions_matched": 8,
1363
+ "decisions_total": 8,
1364
+ "decision_agreement": 1.0,
1365
+ "reference_margin_min": 0.25,
1366
+ "reference_margin_median": 0.42578125,
1367
+ "flips": [],
1368
+ "decisions": [
1369
+ {
1370
+ "record": 0,
1371
+ "question": 0,
1372
+ "reference_choice": 0,
1373
+ "candidate_choice": 0,
1374
+ "matched": true,
1375
+ "reference_margin": 0.69921875,
1376
+ "logits_max_abs": 0.0,
1377
+ "margin_exceeds_twice_error": true
1378
+ },
1379
+ {
1380
+ "record": 0,
1381
+ "question": 1,
1382
+ "reference_choice": 0,
1383
+ "candidate_choice": 0,
1384
+ "matched": true,
1385
+ "reference_margin": 0.392578125,
1386
+ "logits_max_abs": 0.0,
1387
+ "margin_exceeds_twice_error": true
1388
+ },
1389
+ {
1390
+ "record": 0,
1391
+ "question": 2,
1392
+ "reference_choice": 0,
1393
+ "candidate_choice": 0,
1394
+ "matched": true,
1395
+ "reference_margin": 0.25,
1396
+ "logits_max_abs": 0.0,
1397
+ "margin_exceeds_twice_error": true
1398
+ },
1399
+ {
1400
+ "record": 0,
1401
+ "question": 3,
1402
+ "reference_choice": 0,
1403
+ "candidate_choice": 0,
1404
+ "matched": true,
1405
+ "reference_margin": 0.48828125,
1406
+ "logits_max_abs": 0.0,
1407
+ "margin_exceeds_twice_error": true
1408
+ },
1409
+ {
1410
+ "record": 0,
1411
+ "question": 4,
1412
+ "reference_choice": 0,
1413
+ "candidate_choice": 0,
1414
+ "matched": true,
1415
+ "reference_margin": 0.58984375,
1416
+ "logits_max_abs": 0.0,
1417
+ "margin_exceeds_twice_error": true
1418
+ },
1419
+ {
1420
+ "record": 0,
1421
+ "question": 5,
1422
+ "reference_choice": 0,
1423
+ "candidate_choice": 0,
1424
+ "matched": true,
1425
+ "reference_margin": 0.4296875,
1426
+ "logits_max_abs": 0.0,
1427
+ "margin_exceeds_twice_error": true
1428
+ },
1429
+ {
1430
+ "record": 0,
1431
+ "question": 6,
1432
+ "reference_choice": 0,
1433
+ "candidate_choice": 0,
1434
+ "matched": true,
1435
+ "reference_margin": 0.28515625,
1436
+ "logits_max_abs": 0.0,
1437
+ "margin_exceeds_twice_error": true
1438
+ },
1439
+ {
1440
+ "record": 0,
1441
+ "question": 7,
1442
+ "reference_choice": 0,
1443
+ "candidate_choice": 0,
1444
+ "matched": true,
1445
+ "reference_margin": 0.421875,
1446
+ "logits_max_abs": 0.0,
1447
+ "margin_exceeds_twice_error": true
1448
+ }
1449
+ ]
1450
+ },
1451
+ "resident_vram_bytes": 55004415488,
1452
+ "peak_vram_bytes": 55265243648
1453
+ },
1454
+ {
1455
+ "case": 2,
1456
+ "tokens": 2270,
1457
+ "total_ms": {
1458
+ "mean": 207.1281478740275,
1459
+ "median": 206.86872629448771,
1460
+ "p95": 211.94536611437798,
1461
+ "min": 203.42395268380642,
1462
+ "max": 216.9617060571909
1463
+ },
1464
+ "backbone_ms": {
1465
+ "mean": 202.96321356048185,
1466
+ "median": 203.08450981974602,
1467
+ "p95": 204.29294928908348,
1468
+ "min": 200.65308827906847,
1469
+ "max": 204.85719107091427
1470
+ },
1471
+ "hidden_error": {
1472
+ "max_abs": 0.0,
1473
+ "relative_l2": 0.0
1474
+ },
1475
+ "drift": {
1476
+ "logits_max_abs": 0.0,
1477
+ "logits_relative_l2": 0.0,
1478
+ "decisions_matched": 8,
1479
+ "decisions_total": 8,
1480
+ "decision_agreement": 1.0,
1481
+ "reference_margin_min": 0.01953125,
1482
+ "reference_margin_median": 0.12109375,
1483
+ "flips": [],
1484
+ "decisions": [
1485
+ {
1486
+ "record": 0,
1487
+ "question": 0,
1488
+ "reference_choice": 0,
1489
+ "candidate_choice": 0,
1490
+ "matched": true,
1491
+ "reference_margin": 0.4765625,
1492
+ "logits_max_abs": 0.0,
1493
+ "margin_exceeds_twice_error": true
1494
+ },
1495
+ {
1496
+ "record": 0,
1497
+ "question": 1,
1498
+ "reference_choice": 0,
1499
+ "candidate_choice": 0,
1500
+ "matched": true,
1501
+ "reference_margin": 0.306640625,
1502
+ "logits_max_abs": 0.0,
1503
+ "margin_exceeds_twice_error": true
1504
+ },
1505
+ {
1506
+ "record": 0,
1507
+ "question": 2,
1508
+ "reference_choice": 0,
1509
+ "candidate_choice": 0,
1510
+ "matched": true,
1511
+ "reference_margin": 0.08203125,
1512
+ "logits_max_abs": 0.0,
1513
+ "margin_exceeds_twice_error": true
1514
+ },
1515
+ {
1516
+ "record": 0,
1517
+ "question": 3,
1518
+ "reference_choice": 0,
1519
+ "candidate_choice": 0,
1520
+ "matched": true,
1521
+ "reference_margin": 0.1171875,
1522
+ "logits_max_abs": 0.0,
1523
+ "margin_exceeds_twice_error": true
1524
+ },
1525
+ {
1526
+ "record": 0,
1527
+ "question": 4,
1528
+ "reference_choice": 0,
1529
+ "candidate_choice": 0,
1530
+ "matched": true,
1531
+ "reference_margin": 0.01953125,
1532
+ "logits_max_abs": 0.0,
1533
+ "margin_exceeds_twice_error": true
1534
+ },
1535
+ {
1536
+ "record": 0,
1537
+ "question": 5,
1538
+ "reference_choice": 0,
1539
+ "candidate_choice": 0,
1540
+ "matched": true,
1541
+ "reference_margin": 0.05859375,
1542
+ "logits_max_abs": 0.0,
1543
+ "margin_exceeds_twice_error": true
1544
+ },
1545
+ {
1546
+ "record": 0,
1547
+ "question": 6,
1548
+ "reference_choice": 3,
1549
+ "candidate_choice": 3,
1550
+ "matched": true,
1551
+ "reference_margin": 0.125,
1552
+ "logits_max_abs": 0.0,
1553
+ "margin_exceeds_twice_error": true
1554
+ },
1555
+ {
1556
+ "record": 0,
1557
+ "question": 7,
1558
+ "reference_choice": 0,
1559
+ "candidate_choice": 0,
1560
+ "matched": true,
1561
+ "reference_margin": 0.265625,
1562
+ "logits_max_abs": 0.0,
1563
+ "margin_exceeds_twice_error": true
1564
+ }
1565
+ ]
1566
+ },
1567
+ "resident_vram_bytes": 55004415488,
1568
+ "peak_vram_bytes": 55398181376
1569
+ },
1570
+ {
1571
+ "case": 3,
1572
+ "tokens": 298,
1573
+ "total_ms": {
1574
+ "mean": 64.96169349799554,
1575
+ "median": 64.82085539028049,
1576
+ "p95": 65.15412591397762,
1577
+ "min": 64.52513858675957,
1578
+ "max": 68.82472150027752
1579
+ },
1580
+ "backbone_ms": {
1581
+ "mean": 61.811730451881886,
1582
+ "median": 61.80641520768404,
1583
+ "p95": 62.05604784190655,
1584
+ "min": 61.49221956729889,
1585
+ "max": 62.236362136900425
1586
+ },
1587
+ "hidden_error": {
1588
+ "max_abs": 0.0,
1589
+ "relative_l2": 0.0
1590
+ },
1591
+ "drift": {
1592
+ "logits_max_abs": 0.0,
1593
+ "logits_relative_l2": 0.0,
1594
+ "decisions_matched": 2,
1595
+ "decisions_total": 2,
1596
+ "decision_agreement": 1.0,
1597
+ "reference_margin_min": 4.31640625,
1598
+ "reference_margin_median": 4.353515625,
1599
+ "flips": [],
1600
+ "decisions": [
1601
+ {
1602
+ "record": 0,
1603
+ "question": 0,
1604
+ "reference_choice": 3,
1605
+ "candidate_choice": 3,
1606
+ "matched": true,
1607
+ "reference_margin": 4.390625,
1608
+ "logits_max_abs": 0.0,
1609
+ "margin_exceeds_twice_error": true
1610
+ },
1611
+ {
1612
+ "record": 0,
1613
+ "question": 1,
1614
+ "reference_choice": 1,
1615
+ "candidate_choice": 1,
1616
+ "matched": true,
1617
+ "reference_margin": 4.31640625,
1618
+ "logits_max_abs": 0.0,
1619
+ "margin_exceeds_twice_error": true
1620
+ }
1621
+ ]
1622
+ },
1623
+ "resident_vram_bytes": 55004415488,
1624
+ "peak_vram_bytes": 55056573440
1625
+ },
1626
+ {
1627
+ "case": 4,
1628
+ "tokens": 303,
1629
+ "total_ms": {
1630
+ "mean": 64.5604116221269,
1631
+ "median": 64.56589605659246,
1632
+ "p95": 64.78502973914146,
1633
+ "min": 64.3211891874671,
1634
+ "max": 64.87731356173754
1635
+ },
1636
+ "backbone_ms": {
1637
+ "mean": 61.7412888755401,
1638
+ "median": 61.68849114328623,
1639
+ "p95": 62.02105898410082,
1640
+ "min": 61.47781666368246,
1641
+ "max": 62.256986275315285
1642
+ },
1643
+ "hidden_error": {
1644
+ "max_abs": 0.0,
1645
+ "relative_l2": 0.0
1646
+ },
1647
+ "drift": {
1648
+ "logits_max_abs": 0.0,
1649
+ "logits_relative_l2": 0.0,
1650
+ "decisions_matched": 2,
1651
+ "decisions_total": 2,
1652
+ "decision_agreement": 1.0,
1653
+ "reference_margin_min": 2.8984375,
1654
+ "reference_margin_median": 3.73046875,
1655
+ "flips": [],
1656
+ "decisions": [
1657
+ {
1658
+ "record": 0,
1659
+ "question": 0,
1660
+ "reference_choice": 2,
1661
+ "candidate_choice": 2,
1662
+ "matched": true,
1663
+ "reference_margin": 4.5625,
1664
+ "logits_max_abs": 0.0,
1665
+ "margin_exceeds_twice_error": true
1666
+ },
1667
+ {
1668
+ "record": 0,
1669
+ "question": 1,
1670
+ "reference_choice": 0,
1671
+ "candidate_choice": 0,
1672
+ "matched": true,
1673
+ "reference_margin": 2.8984375,
1674
+ "logits_max_abs": 0.0,
1675
+ "margin_exceeds_twice_error": true
1676
+ }
1677
+ ]
1678
+ },
1679
+ "resident_vram_bytes": 55004415488,
1680
+ "peak_vram_bytes": 55058310656
1681
+ },
1682
+ {
1683
+ "case": 5,
1684
+ "tokens": 304,
1685
+ "total_ms": {
1686
+ "mean": 64.69714563960831,
1687
+ "median": 64.54729475080967,
1688
+ "p95": 64.86918963491917,
1689
+ "min": 64.34616167098284,
1690
+ "max": 68.42743512243032
1691
+ },
1692
+ "backbone_ms": {
1693
+ "mean": 61.82925890510281,
1694
+ "median": 61.83003494516015,
1695
+ "p95": 62.04524822533131,
1696
+ "min": 61.482759192585945,
1697
+ "max": 62.447430565953255
1698
+ },
1699
+ "hidden_error": {
1700
+ "max_abs": 0.0,
1701
+ "relative_l2": 0.0
1702
+ },
1703
+ "drift": {
1704
+ "logits_max_abs": 0.0,
1705
+ "logits_relative_l2": 0.0,
1706
+ "decisions_matched": 2,
1707
+ "decisions_total": 2,
1708
+ "decision_agreement": 1.0,
1709
+ "reference_margin_min": 4.2109375,
1710
+ "reference_margin_median": 4.4228515625,
1711
+ "flips": [],
1712
+ "decisions": [
1713
+ {
1714
+ "record": 0,
1715
+ "question": 0,
1716
+ "reference_choice": 0,
1717
+ "candidate_choice": 0,
1718
+ "matched": true,
1719
+ "reference_margin": 4.2109375,
1720
+ "logits_max_abs": 0.0,
1721
+ "margin_exceeds_twice_error": true
1722
+ },
1723
+ {
1724
+ "record": 0,
1725
+ "question": 1,
1726
+ "reference_choice": 1,
1727
+ "candidate_choice": 1,
1728
+ "matched": true,
1729
+ "reference_margin": 4.634765625,
1730
+ "logits_max_abs": 0.0,
1731
+ "margin_exceeds_twice_error": true
1732
+ }
1733
+ ]
1734
+ },
1735
+ "resident_vram_bytes": 55004415488,
1736
+ "peak_vram_bytes": 55058413056
1737
+ },
1738
+ {
1739
+ "case": 6,
1740
+ "tokens": 312,
1741
+ "total_ms": {
1742
+ "mean": 65.49102198332548,
1743
+ "median": 65.15130214393139,
1744
+ "p95": 67.28412676602602,
1745
+ "min": 64.53924346715212,
1746
+ "max": 69.64690797030926
1747
+ },
1748
+ "backbone_ms": {
1749
+ "mean": 62.528794786582395,
1750
+ "median": 62.569474801421165,
1751
+ "p95": 63.048550859093666,
1752
+ "min": 62.03409377485514,
1753
+ "max": 63.295284286141396
1754
+ },
1755
+ "hidden_error": {
1756
+ "max_abs": 0.0,
1757
+ "relative_l2": 0.0
1758
+ },
1759
+ "drift": {
1760
+ "logits_max_abs": 0.0,
1761
+ "logits_relative_l2": 0.0,
1762
+ "decisions_matched": 2,
1763
+ "decisions_total": 2,
1764
+ "decision_agreement": 1.0,
1765
+ "reference_margin_min": 1.7109375,
1766
+ "reference_margin_median": 2.55859375,
1767
+ "flips": [],
1768
+ "decisions": [
1769
+ {
1770
+ "record": 0,
1771
+ "question": 0,
1772
+ "reference_choice": 2,
1773
+ "candidate_choice": 2,
1774
+ "matched": true,
1775
+ "reference_margin": 3.40625,
1776
+ "logits_max_abs": 0.0,
1777
+ "margin_exceeds_twice_error": true
1778
+ },
1779
+ {
1780
+ "record": 0,
1781
+ "question": 1,
1782
+ "reference_choice": 0,
1783
+ "candidate_choice": 0,
1784
+ "matched": true,
1785
+ "reference_margin": 1.7109375,
1786
+ "logits_max_abs": 0.0,
1787
+ "margin_exceeds_twice_error": true
1788
+ }
1789
+ ]
1790
+ },
1791
+ "resident_vram_bytes": 55004415488,
1792
+ "peak_vram_bytes": 55060240896
1793
+ }
1794
+ ],
1795
+ "conversion": null,
1796
+ "shared_input_exact": null,
1797
+ "patch_statistics": {
1798
+ "rms_calls": 43456,
1799
+ "swiglu_calls": 28672,
1800
+ "residual_calls": 28672,
1801
+ "delta_calls": 21504,
1802
+ "shared_quant_calls": 0,
1803
+ "shared_attn_quant_calls": 0,
1804
+ "fallbacks": 0
1805
+ }
1806
+ }
1807
+ },
1808
+ "quality_scope": "BF16 agreement only, not task accuracy or production validation",
1809
+ "records_source": "synthetic",
1810
+ "bracketed": true
1811
+ }
results/calibration.json ADDED
The diff for this file is too large to render. See raw diff
 
runtime/clef-bench.py ADDED
@@ -0,0 +1,479 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Profile Cloudflare Clef backbone and joint-head latency."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import importlib.util
8
+ import json
9
+ import os
10
+ import statistics
11
+ import sys
12
+ import time
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+
17
+ def _prepend_env_path(name: str, value: Path) -> None:
18
+ if not value.exists():
19
+ return
20
+ current = os.environ.get(name)
21
+ value_text = str(value)
22
+ if not current:
23
+ os.environ[name] = value_text
24
+ return
25
+ parts = current.split(os.pathsep)
26
+ if value_text not in parts:
27
+ os.environ[name] = os.pathsep.join([value_text, current])
28
+
29
+
30
+ def _configure_cuda_compiler_env() -> None:
31
+ cuda_home = os.environ.get("CUDA_HOME")
32
+ candidates = [Path(cuda_home)] if cuda_home else []
33
+ candidates.extend([Path("/usr/local/cuda"), Path("/usr/local/cuda-13.0"), Path("/usr/local/cuda-12.8")])
34
+ cuda_root = next((path.resolve() for path in candidates if path.exists()), None)
35
+ if cuda_root is None:
36
+ return
37
+
38
+ os.environ.setdefault("CUDA_HOME", str(cuda_root))
39
+ ptxas = cuda_root / "bin" / "ptxas"
40
+ if ptxas.exists():
41
+ os.environ.setdefault("TRITON_PTXAS_PATH", str(ptxas))
42
+ _prepend_env_path("PATH", ptxas.parent)
43
+
44
+ target = cuda_root / "targets" / "x86_64-linux"
45
+ _prepend_env_path("CPATH", target / "include")
46
+ _prepend_env_path("LIBRARY_PATH", target / "lib" / "stubs")
47
+
48
+
49
+ _configure_cuda_compiler_env()
50
+
51
+ import torch
52
+
53
+
54
+ DEFAULT_MODEL = "Cloudflare/clef"
55
+
56
+
57
+ def parse_args() -> argparse.Namespace:
58
+ parser = argparse.ArgumentParser(description=__doc__)
59
+ parser.add_argument("--model", default=DEFAULT_MODEL)
60
+ parser.add_argument("--local-files-only", action="store_true")
61
+ parser.add_argument("--device", default="cuda")
62
+ parser.add_argument("--dtype", choices=["bfloat16", "float16", "float32"], default="bfloat16")
63
+ parser.add_argument("--records", type=int, default=1)
64
+ parser.add_argument("--questions", type=int, default=8)
65
+ parser.add_argument("--choices", type=int, default=4)
66
+ parser.add_argument("--state-words", type=int, default=256)
67
+ parser.add_argument("--max-length", type=int, default=16384)
68
+ parser.add_argument("--warmup", type=int, default=2)
69
+ parser.add_argument("--repeat", type=int, default=10)
70
+ parser.add_argument("--profile", action="store_true")
71
+ parser.add_argument("--profile-stages", action="store_true")
72
+ parser.add_argument("--capture-nsys", action="store_true")
73
+ parser.add_argument("--profile-iterations", type=int, default=3)
74
+ parser.add_argument("--image-size", type=int, default=0)
75
+ parser.add_argument("--image-count", type=int, default=1)
76
+ parser.add_argument("--vision-backend", choices=["reference", "no-sync", "cute", "cute-fused", "cute-graph"], default="reference")
77
+ parser.add_argument("--compare-vision", action="store_true")
78
+ parser.add_argument("--text-backend", choices=["reference", "cute-rms", "cute-swiglu", "cute", "cute-gva", "cute-gates", "cute-all", "cute-heads"], default="reference")
79
+ parser.add_argument("--compare-text", action="store_true")
80
+ parser.add_argument("--compile-head", action="store_true")
81
+ parser.add_argument(
82
+ "--linear-attn-backend",
83
+ choices=["auto", "torch"],
84
+ default="auto",
85
+ help="Use Transformers' selected Qwen linear-attention kernels, or force the torch fallback.",
86
+ )
87
+ parser.add_argument("--output", type=Path, default=Path("clef-bench.json"))
88
+ return parser.parse_args()
89
+
90
+
91
+ def dtype_from_name(name: str) -> torch.dtype:
92
+ return {
93
+ "bfloat16": torch.bfloat16,
94
+ "float16": torch.float16,
95
+ "float32": torch.float32,
96
+ }[name]
97
+
98
+
99
+ def load_joint_schema_module(model_path: Path):
100
+ module_path = model_path / "joint_schema_model.py"
101
+ spec = importlib.util.spec_from_file_location("clef_joint_schema_model", module_path)
102
+ if spec is None or spec.loader is None:
103
+ raise RuntimeError(f"Could not import {module_path}")
104
+ module = importlib.util.module_from_spec(spec)
105
+ sys.modules[spec.name] = module
106
+ spec.loader.exec_module(module)
107
+ return module
108
+
109
+
110
+ def make_record(index: int, questions: int, choices: int, state_words: int) -> dict[str, Any]:
111
+ words = [
112
+ "invoice",
113
+ "router",
114
+ "customer",
115
+ "risk",
116
+ "policy",
117
+ "payment",
118
+ "service",
119
+ "contract",
120
+ "evidence",
121
+ "priority",
122
+ ]
123
+ state = {
124
+ "case_id": f"bench-{index}",
125
+ "summary": " ".join(words[i % len(words)] for i in range(state_words)),
126
+ "amount": 1250 + index,
127
+ "currency": "USD",
128
+ "signals": {
129
+ "escalated": index % 2 == 0,
130
+ "region": "us",
131
+ "latency_ms": 217,
132
+ },
133
+ }
134
+ schema = {}
135
+ for q_idx in range(questions):
136
+ schema[f"field_{q_idx:03d}"] = {
137
+ "type": "choice",
138
+ "instructions": f"Choose the best route for field {q_idx}.",
139
+ "criteria": {
140
+ f"option_{choice_idx:02d}": f"Evidence pattern {choice_idx} for field {q_idx}."
141
+ for choice_idx in range(choices)
142
+ },
143
+ }
144
+ return {"id": f"bench-{index}", "state": state, "questions": schema}
145
+
146
+
147
+ def sync(device: torch.device) -> None:
148
+ if device.type == "cuda":
149
+ torch.cuda.synchronize(device)
150
+
151
+
152
+ def timed(fn, device: torch.device) -> tuple[Any, float]:
153
+ sync(device)
154
+ start = time.perf_counter()
155
+ value = fn()
156
+ sync(device)
157
+ return value, time.perf_counter() - start
158
+
159
+
160
+ def percentile(values: list[float], pct: float) -> float:
161
+ if not values:
162
+ return float("nan")
163
+ ordered = sorted(values)
164
+ index = min(len(ordered) - 1, max(0, round((len(ordered) - 1) * pct)))
165
+ return ordered[index]
166
+
167
+
168
+ def summarize_ms(values: list[float]) -> dict[str, float]:
169
+ return {
170
+ "mean": statistics.fmean(values) * 1000,
171
+ "median": statistics.median(values) * 1000,
172
+ "p95": percentile(values, 0.95) * 1000,
173
+ "min": min(values) * 1000,
174
+ "max": max(values) * 1000,
175
+ }
176
+
177
+
178
+ def force_torch_linear_attention(text_model: torch.nn.Module) -> int:
179
+ module = importlib.import_module(text_model.__class__.__module__)
180
+ rms_norm_gated_cls = getattr(
181
+ module,
182
+ "Qwen3_5RMSNormGated",
183
+ getattr(module, "Qwen3NextRMSNormGated", None),
184
+ )
185
+ if rms_norm_gated_cls is None:
186
+ raise RuntimeError("Could not find the Qwen gated RMSNorm fallback class")
187
+ replaced = 0
188
+ for layer in getattr(text_model, "layers", []):
189
+ linear_attn = getattr(layer, "linear_attn", None)
190
+ if linear_attn is None:
191
+ continue
192
+ linear_attn.causal_conv1d_fn = None
193
+ linear_attn.causal_conv1d_update = module.torch_causal_conv1d_update
194
+ linear_attn.chunk_gated_delta_rule = module.torch_chunk_gated_delta_rule
195
+ linear_attn.recurrent_gated_delta_rule = module.torch_recurrent_gated_delta_rule
196
+ if not isinstance(linear_attn.norm, rms_norm_gated_cls):
197
+ original_norm = linear_attn.norm
198
+ replacement = rms_norm_gated_cls(
199
+ linear_attn.head_v_dim,
200
+ eps=linear_attn.layer_norm_epsilon,
201
+ ).to(device=original_norm.weight.device, dtype=original_norm.weight.dtype)
202
+ replacement.weight.data.copy_(original_norm.weight.data)
203
+ linear_attn.norm = replacement
204
+ replaced += 1
205
+ if replaced == 0:
206
+ raise RuntimeError("No Qwen linear-attention layers were found to patch")
207
+ return replaced
208
+
209
+
210
+ def main() -> None:
211
+ args = parse_args()
212
+ if args.compare_text and (args.text_backend == "reference" or args.compare_vision):
213
+ raise ValueError("--compare-text requires a text candidate and cannot be combined with --compare-vision")
214
+ if min(args.repeat, args.warmup, args.profile_iterations) < 1:
215
+ raise ValueError("Repeat, warmup, and profile iterations must be positive")
216
+ if args.linear_attn_backend == "torch" and args.text_backend in {"cute-gva", "cute-all", "cute-heads"}:
217
+ raise ValueError("Native GVA requires the FLA backend, not --linear-attn-backend torch")
218
+ os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
219
+ device = torch.device(args.device)
220
+ dtype = dtype_from_name(args.dtype)
221
+
222
+ from huggingface_hub import snapshot_download
223
+ from transformers.utils.import_utils import (
224
+ is_causal_conv1d_available,
225
+ is_flash_linear_attention_available,
226
+ )
227
+
228
+ model_path = Path(args.model)
229
+ if not model_path.is_dir():
230
+ model_path = Path(snapshot_download(args.model, local_files_only=args.local_files_only))
231
+ joint_schema = load_joint_schema_module(model_path)
232
+ model, processor = joint_schema.load_release_model(model_path, device=device, dtype=dtype)
233
+ if args.compile_head:
234
+ model.head = torch.compile(model.head, mode="reduce-overhead")
235
+
236
+ records = [
237
+ make_record(index, args.questions, args.choices, args.state_words)
238
+ for index in range(args.records)
239
+ ]
240
+ if args.image_size:
241
+ from PIL import Image
242
+ import numpy as np
243
+
244
+ for i, record in enumerate(records):
245
+ rng = np.random.default_rng(1234 + i)
246
+ record["images"] = [
247
+ Image.fromarray(rng.integers(0, 256, (args.image_size, args.image_size, 3), dtype=np.uint8))
248
+ for _ in range(args.image_count)
249
+ ]
250
+ elif args.vision_backend != "reference" or args.compare_vision:
251
+ raise ValueError("Vision benchmarking requires --image-size")
252
+ if args.compare_vision and args.vision_backend == "reference":
253
+ raise ValueError("--compare-vision requires a candidate vision backend")
254
+ encoded, encode_seconds = timed(
255
+ lambda: [
256
+ joint_schema.encode_record(
257
+ processor.tokenizer,
258
+ record,
259
+ max_length=args.max_length,
260
+ processor=processor,
261
+ )
262
+ for record in records
263
+ ],
264
+ device,
265
+ )
266
+ batch = joint_schema.collate_records(encoded, processor.tokenizer.pad_token_id, device)
267
+
268
+ base_model = (
269
+ model.language_model.get_base_model()
270
+ if hasattr(model.language_model, "get_base_model")
271
+ else model.language_model
272
+ )
273
+ text_model = base_model.model
274
+ media = batch.get("media") or {}
275
+ if not media and hasattr(text_model, "language_model"):
276
+ text_model = text_model.language_model
277
+ forced_linear_attn_layers = 0
278
+ if args.linear_attn_backend == "torch":
279
+ forced_linear_attn_layers = force_torch_linear_attention(getattr(text_model, "language_model", text_model))
280
+
281
+ def run_backbone():
282
+ return text_model(
283
+ input_ids=batch["input_ids"],
284
+ attention_mask=batch["attention_mask"],
285
+ use_cache=False,
286
+ return_dict=True,
287
+ **media,
288
+ )
289
+
290
+ def run_once() -> tuple[Any, Any]:
291
+ outputs = run_backbone()
292
+ logits = model.head(
293
+ outputs.last_hidden_state,
294
+ batch["input_ids"],
295
+ batch["attention_mask"],
296
+ batch["records"],
297
+ base_model.get_output_embeddings().weight,
298
+ )
299
+ return outputs, logits
300
+
301
+ vision_comparison = None
302
+ text_comparison = None
303
+ with torch.inference_mode():
304
+ if args.vision_backend != "reference":
305
+ from clef_vision_rope import patch_attention
306
+
307
+ if args.compare_vision:
308
+ for _ in range(args.warmup):
309
+ run_once()
310
+ reference_logits = [[v.clone() for v in r] for r in run_once()[1]]
311
+ reference_times = [timed(run_once, device)[1] for _ in range(args.repeat)]
312
+ grid = media["image_grid_thw"].cpu().tolist()
313
+ lengths = tuple(h * w for t, h, w in grid for _ in range(t))
314
+ def install_vision():
315
+ if args.vision_backend in {"cute-fused", "cute-graph"}:
316
+ from clef_vision_prepared import patch_prepared_vision
317
+ return patch_prepared_vision(base_model.model.visual, capture=args.vision_backend == "cute-graph")
318
+ return patch_attention(base_model.model.visual, rope=args.vision_backend == "cute", lengths=lengths)
319
+
320
+ restore_vision = install_vision()
321
+ if args.compare_vision:
322
+ actual_logits = run_once()[1]
323
+ expected = torch.cat([v.flatten() for r in reference_logits for v in r])
324
+ actual = torch.cat([v.flatten() for r in actual_logits for v in r])
325
+ delta = (actual.float() - expected.float())
326
+ matches = sum(int(a.argmax() == b.argmax()) for ar, br in zip(actual_logits, reference_logits)
327
+ for a, b in zip(ar, br))
328
+ count = sum(len(r) for r in reference_logits)
329
+ vision_comparison = {"reference_total_ms": summarize_ms(reference_times),
330
+ "logits_max_abs": delta.abs().max().item(),
331
+ "logits_relative_l2": (delta.norm() / expected.float().norm().clamp_min(1e-12)).item(),
332
+ "decisions_matched": matches, "decisions_total": count}
333
+ torch.testing.assert_close(actual, expected, atol=0, rtol=0)
334
+ if matches != count:
335
+ raise AssertionError("CuTe vision changed a CLEF decision")
336
+ if args.text_backend != "reference":
337
+ from clef_text_cute import patch_text
338
+ if args.compare_text:
339
+ for _ in range(args.warmup):
340
+ run_once()
341
+ text_expected = torch.cat([v.flatten() for r in run_once()[1] for v in r]).clone()
342
+ text_reference_times = [timed(run_once, device)[1] for _ in range(args.repeat)]
343
+ def install_text():
344
+ return patch_text(getattr(text_model, "language_model", text_model),
345
+ rms=args.text_backend != "cute-swiglu",
346
+ activation=args.text_backend != "cute-rms",
347
+ gva=args.text_backend in {"cute-gva", "cute-all", "cute-heads"},
348
+ gates=args.text_backend in {"cute-gates", "cute-all", "cute-heads"},
349
+ heads=args.text_backend == "cute-heads")
350
+ restore_text = install_text()
351
+ if args.compare_text:
352
+ actual_logits = run_once()[1]
353
+ actual = torch.cat([v.flatten() for r in actual_logits for v in r])
354
+ torch.testing.assert_close(actual, text_expected, atol=0, rtol=0)
355
+ text_comparison = {"reference_total_ms": summarize_ms(text_reference_times),
356
+ "logits_max_abs": float((actual.float() - text_expected.float()).abs().max()),
357
+ "decisions_total": sum(len(r) for r in actual_logits), "exact": True}
358
+ for _ in range(args.warmup):
359
+ run_once()
360
+ sync(device)
361
+
362
+ backbone_times: list[float] = []
363
+ head_times: list[float] = []
364
+ total_times: list[float] = []
365
+ for _ in range(args.repeat):
366
+ sync(device)
367
+ total_start = time.perf_counter()
368
+ outputs, backbone_elapsed = timed(run_backbone, device)
369
+ _, head_elapsed = timed(
370
+ lambda: model.head(
371
+ outputs.last_hidden_state,
372
+ batch["input_ids"],
373
+ batch["attention_mask"],
374
+ batch["records"],
375
+ base_model.get_output_embeddings().weight,
376
+ ),
377
+ device,
378
+ )
379
+ sync(device)
380
+ total_times.append(time.perf_counter() - total_start)
381
+ backbone_times.append(backbone_elapsed)
382
+ head_times.append(head_elapsed)
383
+
384
+ if args.compare_vision:
385
+ candidate_times = [timed(run_once, device)[1] for _ in range(args.repeat)]
386
+ if hasattr(restore_vision, "statistics"):
387
+ vision_comparison["prepared_cache"] = dict(restore_vision.statistics)
388
+ if not restore_vision.statistics["calls"] or restore_vision.statistics["fallbacks"]:
389
+ raise AssertionError("Prepared vision was not used for every candidate call")
390
+ restore_vision()
391
+ reference_after = [timed(run_once, device)[1] for _ in range(args.repeat)]
392
+ vision_comparison["candidate_total_ms"] = summarize_ms(candidate_times)
393
+ vision_comparison["reference_after_total_ms"] = summarize_ms(reference_after)
394
+ vision_comparison["speedup"] = ((statistics.median(reference_times) + statistics.median(reference_after))
395
+ / (2 * statistics.median(candidate_times)))
396
+ install_vision()
397
+
398
+ if args.compare_text:
399
+ candidate_times = [timed(run_once, device)[1] for _ in range(args.repeat)]
400
+ text_comparison["patch_statistics"] = dict(restore_text.statistics)
401
+ restore_text()
402
+ reference_after = [timed(run_once, device)[1] for _ in range(args.repeat)]
403
+ text_comparison["candidate_total_ms"] = summarize_ms(candidate_times)
404
+ text_comparison["reference_after_total_ms"] = summarize_ms(reference_after)
405
+ text_comparison["speedup"] = ((statistics.median(text_reference_times) + statistics.median(reference_after))
406
+ / (2 * statistics.median(candidate_times)))
407
+ restore_text = install_text()
408
+
409
+ profile_table = None
410
+ stage_profile = None
411
+ if args.profile:
412
+ activities = [torch.profiler.ProfilerActivity.CPU]
413
+ if device.type == "cuda":
414
+ activities.append(torch.profiler.ProfilerActivity.CUDA)
415
+ with torch.profiler.profile(
416
+ activities=activities,
417
+ record_shapes=True,
418
+ profile_memory=True,
419
+ with_stack=False,
420
+ ) as prof:
421
+ run_once()
422
+ profile_table = prof.key_averages().table(
423
+ sort_by="cuda_time_total" if device.type == "cuda" else "cpu_time_total",
424
+ row_limit=40,
425
+ )
426
+ if args.profile_stages or args.capture_nsys:
427
+ from clef_text_profile import profile_text
428
+ stage_profile = profile_text(run_once, getattr(text_model, "language_model", text_model),
429
+ args.output.parent / (args.output.stem + "-profile"), head=model.head,
430
+ iterations=args.profile_iterations, capture_nsys=args.capture_nsys)
431
+
432
+ max_memory = None
433
+ if device.type == "cuda":
434
+ max_memory = torch.cuda.max_memory_allocated(device)
435
+
436
+ result = {
437
+ "model": args.model,
438
+ "model_path": str(model_path),
439
+ "device": str(device),
440
+ "device_name": torch.cuda.get_device_name(device) if device.type == "cuda" else None,
441
+ "dtype": args.dtype,
442
+ "torch": torch.__version__,
443
+ "fast_path": {
444
+ "flash_linear_attention": is_flash_linear_attention_available(),
445
+ "causal_conv1d": is_causal_conv1d_available(),
446
+ },
447
+ "linear_attn_backend": args.linear_attn_backend,
448
+ "vision_backend": args.vision_backend,
449
+ "vision_comparison": vision_comparison,
450
+ "text_backend": args.text_backend,
451
+ "text_comparison": text_comparison,
452
+ "forced_linear_attn_layers": forced_linear_attn_layers,
453
+ "shape": {
454
+ "records": args.records,
455
+ "questions": args.questions,
456
+ "choices": args.choices,
457
+ "input_tokens": [len(record.input_ids) for record in encoded],
458
+ "image_size": args.image_size,
459
+ "image_count_per_record": args.image_count if args.image_size else 0,
460
+ },
461
+ "encode_ms": encode_seconds * 1000,
462
+ "latency_ms": {
463
+ "total": summarize_ms(total_times),
464
+ "backbone": summarize_ms(backbone_times),
465
+ "head": summarize_ms(head_times),
466
+ },
467
+ "max_memory_allocated": max_memory,
468
+ "profile_table": profile_table,
469
+ "stage_profile": stage_profile,
470
+ }
471
+ args.output.parent.mkdir(parents=True, exist_ok=True)
472
+ args.output.write_text(json.dumps(result, indent=2) + "\n")
473
+ print(json.dumps({k: v for k, v in result.items() if k not in {"profile_table", "stage_profile"}}, indent=2))
474
+ if profile_table:
475
+ print(profile_table)
476
+
477
+
478
+ if __name__ == "__main__":
479
+ main()
runtime/clef-bootstrap-runtime ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ VENV="${CLEF_VENV:-/workspace/clef-venv}"
5
+ CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}"
6
+ CUDA_HOME="$(readlink -f "${CUDA_HOME}")"
7
+
8
+ export CUDA_HOME
9
+ export PATH="${CUDA_HOME}/bin:${PATH}"
10
+ export CPATH="${CUDA_HOME}/targets/x86_64-linux/include${CPATH:+:${CPATH}}"
11
+ export LIBRARY_PATH="${CUDA_HOME}/targets/x86_64-linux/lib/stubs${LIBRARY_PATH:+:${LIBRARY_PATH}}"
12
+ export TRITON_PTXAS_PATH="${TRITON_PTXAS_PATH:-${CUDA_HOME}/bin/ptxas}"
13
+ export TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-9.0}"
14
+ export MAX_JOBS="${MAX_JOBS:-8}"
15
+
16
+ python -m venv --system-site-packages "${VENV}"
17
+ "${VENV}/bin/python" -m pip install --upgrade pip
18
+ "${VENV}/bin/python" -m pip install --upgrade --no-cache-dir \
19
+ transformers==5.12.1 \
20
+ huggingface_hub \
21
+ safetensors \
22
+ pillow \
23
+ accelerate \
24
+ nvidia-cutlass-dsl==4.8.0 \
25
+ apache-tvm-ffi==0.1.14.post1 \
26
+ flash-linear-attention==0.5.2
27
+ "${VENV}/bin/python" -m pip install --no-cache-dir --no-build-isolation \
28
+ causal-conv1d==1.7.0
29
+
30
+ if [[ "${CLEF_INSTALL_INT8:-0}" == "1" ]]; then
31
+ "${VENV}/bin/python" -m pip install --no-cache-dir --no-deps comfy-kitchen==0.2.31
32
+ fi
33
+
34
+ "${VENV}/bin/python" - <<'PY'
35
+ import torch
36
+ from transformers.utils.import_utils import (
37
+ is_causal_conv1d_available,
38
+ is_flash_linear_attention_available,
39
+ )
40
+
41
+ print("torch", torch.__version__, torch.version.cuda)
42
+ print("flash_linear_attention", is_flash_linear_attention_available())
43
+ print("causal_conv1d", is_causal_conv1d_available())
44
+ PY
runtime/clef-int8-bench.py ADDED
@@ -0,0 +1,236 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Bracket experimental CLEF ConvRot inference with exact fused BF16 baselines."""
3
+
4
+ import argparse
5
+ import gc
6
+ import importlib.util
7
+ import json
8
+ from pathlib import Path
9
+ import time
10
+
11
+ spec = importlib.util.spec_from_file_location("clef_bench", Path(__file__).with_name("clef-bench.py"))
12
+ bench = importlib.util.module_from_spec(spec)
13
+ spec.loader.exec_module(bench)
14
+
15
+ import torch
16
+ from huggingface_hub import snapshot_download
17
+ from clef_text_cute import patch_text
18
+ from clef_text_int8 import quantize_text
19
+ from clef_text_metrics import decision_drift
20
+ from clef_text_profile import profile_text
21
+
22
+
23
+ def semantic_records():
24
+ states = [
25
+ {"summary": "An authenticated customer requests a routine refund for a duplicate charge.",
26
+ "authenticated": True, "duplicate_charge": True, "amount": 24},
27
+ {"summary": "An unverified caller requests an urgent transfer and asks to bypass approval.",
28
+ "authenticated": False, "duplicate_charge": False, "amount": 95000},
29
+ {"summary": "A network incident is resolved. The customer confirms service has recovered.",
30
+ "latency_ms": 14, "packet_loss_percent": 0, "active_incident": False},
31
+ {"summary": "Latency is high, but the report also says the outage is resolved. Evidence conflicts.",
32
+ "latency_ms": 2000, "packet_loss_percent": 35, "active_incident": False},
33
+ ]
34
+ return [{"id": f"semantic-{i}", "state": state, "questions": {
35
+ "route": {"type": "choice", "instructions": "Select the most appropriate handling route.",
36
+ "criteria": {"routine": "Low-risk routine support with sufficient verified evidence.",
37
+ "review": "Conflicting evidence or an unverified high-risk request.",
38
+ "incident": "An ongoing service reliability incident.",
39
+ "close": "A resolved issue requiring no further action."}},
40
+ "urgency": {"type": "choice", "instructions": "Assess urgency from evidence, not requested urgency.",
41
+ "criteria": {"low": "No current harm or only a routine small issue.",
42
+ "high": "Substantial financial risk or current severe service degradation."}},
43
+ }} for i, state in enumerate(states)]
44
+
45
+
46
+ def cpu_outputs(outputs, logits):
47
+ return outputs.last_hidden_state.detach().cpu(), [[v.detach().float().cpu().flatten().tolist() for v in r] for r in logits]
48
+
49
+
50
+ def hidden_error(actual, expected):
51
+ a, b = actual.float(), expected.float()
52
+ diff = a - b
53
+ if not torch.isfinite(a).all() or not torch.isfinite(b).all():
54
+ raise ValueError("Non-finite CLEF hidden states")
55
+ return {"max_abs": float(diff.abs().max()),
56
+ "relative_l2": float(diff.norm() / b.norm().clamp_min(1e-12))}
57
+
58
+
59
+ def main():
60
+ p = argparse.ArgumentParser(description=__doc__)
61
+ p.add_argument("--model", default="Cloudflare/clef")
62
+ p.add_argument("--state-words", type=int, nargs="+", default=[32, 256, 1024])
63
+ p.add_argument("--records-file", type=Path, help="JSON array of real state/questions records; replaces synthetic probes")
64
+ p.add_argument("--scope", choices=["mlp-gate-up", "mlp", "all"], nargs="+", default=["mlp", "all"])
65
+ p.add_argument("--backend", choices=["cute", "torch"], default="cute")
66
+ p.add_argument("--calibration-index", type=Path, help="Indexed calibrated ConvRot linear overrides")
67
+ p.add_argument("--tile", type=int, nargs=2, default=[128, 256])
68
+ p.add_argument("--cluster", type=int, nargs=2, default=[1, 1])
69
+ p.add_argument("--heads", action="store_true", help="Include experimental exact 256-wide Q/K norm fusion")
70
+ p.add_argument("--share-mlp-input", action="store_true", help="Rotate/quantize the gate/up input only once")
71
+ p.add_argument("--share-attn-input", action="store_true", help="Rotate/quantize the GatedDeltaNet QKV/Z input only once")
72
+ p.add_argument("--warmup", type=int, default=3)
73
+ p.add_argument("--repeat", type=int, default=30)
74
+ p.add_argument("--max-length", type=int, default=16384)
75
+ p.add_argument("--profile", action="store_true")
76
+ p.add_argument("--capture-nsys", action="store_true")
77
+ p.add_argument("--profile-mode", choices=["bf16_before", "mlp-gate-up", "mlp", "all", "bf16_after"], default="all")
78
+ p.add_argument("--profile-case", type=int, default=1)
79
+ p.add_argument("--profile-iterations", type=int, default=3)
80
+ p.add_argument("--output", type=Path, default=Path("clef-int8-bench.json"))
81
+ args = p.parse_args()
82
+ if args.repeat < 1 or args.warmup < 1 or args.profile_iterations < 1:
83
+ p.error("Repeat, warmup, and profile iterations must be positive")
84
+ path = Path(args.model)
85
+ if not path.is_dir():
86
+ path = Path(snapshot_download(args.model, local_files_only=True))
87
+ joint = bench.load_joint_schema_module(path)
88
+ device = torch.device("cuda")
89
+ if torch.cuda.get_device_capability(device) != (9, 0):
90
+ raise ValueError("This experiment requires a Hopper SM90 GPU")
91
+ if args.records_file:
92
+ records = json.loads(args.records_file.read_text())
93
+ if not isinstance(records, list) or not records or any(
94
+ not isinstance(r, dict) or "state" not in r or "questions" not in r for r in records):
95
+ raise ValueError("Expected a nonempty JSON array of state/questions records")
96
+ else:
97
+ records = [bench.make_record(i, 8, 4, words) for i, words in enumerate(args.state_words)] + semantic_records()
98
+ if (args.profile or args.capture_nsys) and not 0 <= args.profile_case < len(records):
99
+ p.error("Profile case must select an existing record")
100
+ modes = [scope for scope in ("mlp-gate-up", "mlp", "all") if scope in args.scope]
101
+ if args.calibration_index is not None and len(modes) != 1:
102
+ p.error("A calibrated checkpoint requires exactly one matching --scope")
103
+ if args.calibration_index is not None:
104
+ from clef_int8_calibration import source_digest
105
+ metadata = json.loads(args.calibration_index.read_text())["metadata"]
106
+ if metadata.get("source_digest") != source_digest(path):
107
+ p.error("Calibrated weights were made from a different model/config/code manifest")
108
+ if (args.profile or args.capture_nsys) and args.profile_mode not in {"bf16_before", "bf16_after", *modes}:
109
+ p.error("Profile mode must be selected by --scope")
110
+ result = {"model_path": str(path), "torch": torch.__version__, "device": torch.cuda.get_device_name(),
111
+ "backend": args.backend, "heads": args.heads, "share_mlp_input": args.share_mlp_input,
112
+ "share_attn_input": args.share_attn_input, "modes": {},
113
+ "quality_scope": "BF16 agreement only, not task accuracy or production validation",
114
+ "records_source": "external" if args.records_file else "synthetic", "bracketed": False}
115
+ args.output.parent.mkdir(parents=True, exist_ok=True)
116
+
117
+ def save():
118
+ args.output.write_text(json.dumps(result, indent=2) + "\n")
119
+
120
+ with torch.inference_mode():
121
+ model, processor = joint.load_release_model(path, device=device, dtype=torch.bfloat16)
122
+ batches, labels = [], []
123
+ for i, record in enumerate(records):
124
+ encoded = joint.encode_record(processor.tokenizer, record, processor=processor, max_length=args.max_length)
125
+ batches.append(joint.collate_records([encoded], processor.tokenizer.pad_token_id, device))
126
+ labels.append({"case": i, "tokens": len(encoded.input_ids)})
127
+ references = []
128
+
129
+ def components(model):
130
+ base = model.language_model.get_base_model() if hasattr(model.language_model, "get_base_model") else model.language_model
131
+ return base, getattr(base.model, "language_model", base.model)
132
+
133
+ def run_once(model, base, text, batch):
134
+ outputs = text(input_ids=batch["input_ids"], attention_mask=batch["attention_mask"],
135
+ use_cache=False, return_dict=True)
136
+ logits = model.head(outputs.last_hidden_state, batch["input_ids"], batch["attention_mask"],
137
+ batch["records"], base.get_output_embeddings().weight)
138
+ return outputs, logits
139
+
140
+ base, text = components(model)
141
+ for batch in batches:
142
+ references.append(cpu_outputs(*run_once(model, base, text, batch)))
143
+ restore = patch_text(text, gva=True, gates=True, heads=args.heads, share_mlp_quant=args.share_mlp_input,
144
+ share_attn_quant=args.share_attn_input)
145
+
146
+ def measure_mode(label, conversion=None, quant_references=None):
147
+ rows = []
148
+ for i, batch in enumerate(batches):
149
+ fn = lambda batch=batch: run_once(model, base, text, batch)
150
+ hidden, logits = cpu_outputs(*fn())
151
+ if quant_references is not None:
152
+ quant_hidden, quant_logits = quant_references[i]
153
+ torch.testing.assert_close(hidden, quant_hidden, atol=0, rtol=0)
154
+ if decision_drift(logits, quant_logits)["logits_max_abs"] != 0:
155
+ raise AssertionError("Shared MLP input changed quantized CLEF logits")
156
+ ref_hidden, ref_logits = references[i]
157
+ drift = decision_drift(logits, ref_logits)
158
+ if label.startswith("bf16"):
159
+ torch.testing.assert_close(hidden, ref_hidden, atol=0, rtol=0)
160
+ if drift["logits_max_abs"] != 0:
161
+ raise AssertionError("Exact text fusions changed CLEF logits")
162
+ error = hidden_error(hidden, ref_hidden)
163
+ del hidden
164
+ for _ in range(args.warmup):
165
+ fn()
166
+ torch.cuda.synchronize()
167
+ torch.cuda.reset_peak_memory_stats()
168
+ times = [bench.timed(fn, device)[1] for _ in range(args.repeat)]
169
+ backbone_fn = lambda batch=batch: text(input_ids=batch["input_ids"], attention_mask=batch["attention_mask"],
170
+ use_cache=False, return_dict=True)
171
+ backbone = [bench.timed(backbone_fn, device)[1] for _ in range(args.repeat)]
172
+ row = {**labels[i], "total_ms": bench.summarize_ms(times),
173
+ "backbone_ms": bench.summarize_ms(backbone), "hidden_error": error, "drift": drift,
174
+ "resident_vram_bytes": torch.cuda.memory_allocated(),
175
+ "peak_vram_bytes": torch.cuda.max_memory_allocated()}
176
+ rows.append(row)
177
+ print(json.dumps({"mode": label, "case": i, "total_ms": row["total_ms"]["median"],
178
+ "matched": drift["decisions_matched"], "decisions": drift["decisions_total"],
179
+ "logits_max_abs": drift["logits_max_abs"]}), flush=True)
180
+ result["modes"][label] = {"cases": rows, "conversion": conversion,
181
+ "shared_input_exact": True if quant_references is not None else None,
182
+ "patch_statistics": dict(restore.statistics)}
183
+ save()
184
+ if label == args.profile_mode and (args.profile or args.capture_nsys):
185
+ batch = batches[args.profile_case]
186
+ result["modes"][label]["profile"] = profile_text(
187
+ lambda: run_once(model, base, text, batch), text,
188
+ args.output.parent / (args.output.stem + "-" + label + "-profile"), head=model.head,
189
+ iterations=args.profile_iterations, capture_nsys=args.capture_nsys)
190
+ save()
191
+
192
+ measure_mode("bf16_before")
193
+ for scope in modes:
194
+ torch.cuda.synchronize()
195
+ started = time.perf_counter()
196
+ conversion = quantize_text(text, scope=scope, backend=args.backend, tile=args.tile, cluster=args.cluster,
197
+ calibration_index=args.calibration_index)
198
+ conversion["seconds"] = time.perf_counter() - started
199
+ quant_references = None
200
+ if args.share_mlp_input or args.share_attn_input:
201
+ restore()
202
+ del restore
203
+ unshared = patch_text(text, gva=True, gates=True, heads=args.heads)
204
+ try:
205
+ quant_references = [cpu_outputs(*run_once(model, base, text, batch)) for batch in batches]
206
+ finally:
207
+ unshared()
208
+ del unshared
209
+ restore = patch_text(text, gva=True, gates=True, heads=args.heads,
210
+ share_mlp_quant=args.share_mlp_input, share_attn_quant=args.share_attn_input)
211
+ measure_mode(scope, conversion, quant_references)
212
+ # Release original forward closures before reloading, rather than retaining two full models.
213
+ restore()
214
+ del restore, model, base, text, processor
215
+ gc.collect()
216
+ torch.cuda.empty_cache()
217
+ model, processor = joint.load_release_model(path, device=device, dtype=torch.bfloat16)
218
+ base, text = components(model)
219
+ restore = patch_text(text, gva=True, gates=True, heads=args.heads, share_mlp_quant=args.share_mlp_input,
220
+ share_attn_quant=args.share_attn_input)
221
+ try:
222
+ measure_mode("bf16_after")
223
+ finally:
224
+ restore()
225
+ for scope in modes:
226
+ for before, candidate, after in zip(result["modes"]["bf16_before"]["cases"],
227
+ result["modes"][scope]["cases"],
228
+ result["modes"]["bf16_after"]["cases"], strict=True):
229
+ reference_ms = (before["total_ms"]["median"] + after["total_ms"]["median"]) / 2
230
+ candidate["speedup_vs_fused_bf16"] = reference_ms / candidate["total_ms"]["median"]
231
+ result["bracketed"] = True
232
+ save()
233
+
234
+
235
+ if __name__ == "__main__":
236
+ main()
runtime/clef-int8-calibrate.py ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Calibrate ConvRot weights, export indexed shards, and evaluate untouched records."""
3
+
4
+ import argparse
5
+ import gc
6
+ import importlib.util
7
+ import json
8
+ from pathlib import Path
9
+ import random
10
+ import time
11
+
12
+ spec = importlib.util.spec_from_file_location("clef_int8_bench", Path(__file__).with_name("clef-int8-bench.py"))
13
+ helpers = importlib.util.module_from_spec(spec)
14
+ spec.loader.exec_module(helpers)
15
+ bench = helpers.bench
16
+
17
+ import torch
18
+ from huggingface_hub import snapshot_download
19
+ from clef_text_cute import patch_text
20
+ from clef_text_int8 import quantize_text
21
+ from clef_text_metrics import decision_drift
22
+ from clef_int8_calibration import (CalibrationWriter, calibrate_linear, capture_samples,
23
+ projection_names, records_digest, source_digest)
24
+
25
+
26
+ def corpus(seed, count):
27
+ rng = random.Random(seed)
28
+ topics = ["duplicate charge", "failed delivery", "service outage", "suspicious transfer",
29
+ "contract renewal", "account access", "payment dispute", "network latency"]
30
+ return [{"id": f"cal-{seed}-{i}", "state": {
31
+ "case_id": f"case-{seed}-{i}", "request": topics[i % len(topics)],
32
+ "summary": " ".join(rng.choice([
33
+ "The customer confirmed the incident is resolved.", "The identity check has not been completed.",
34
+ "The evidence is contradictory and needs independent review.", "An automated alert indicates ongoing harm.",
35
+ "The request follows the standard policy and approval process.", "The requester wants to bypass approval.",
36
+ "The record includes verified supporting documentation.", "No evidence was provided for the claim."
37
+ ]) for _ in range(rng.choice([1, 3, 8, 24]))),
38
+ "amount": rng.choice([0, 8, 99, 1500, 18000, 125000]),
39
+ "authenticated": bool(rng.getrandbits(1)), "risk_score": round(rng.random(), 3),
40
+ "latency_ms": rng.choice([8, 95, 700, 3000]), "active_incident": bool(rng.getrandbits(1)),
41
+ }, "questions": {
42
+ "route": {"type": "choice", "instructions": "Choose the handling route using the available evidence.",
43
+ "criteria": {"routine": "A verified, low-risk routine support request.",
44
+ "review": "Contradictory or incomplete evidence.",
45
+ "escalate": "Substantial current financial risk or active service harm.",
46
+ "close": "The issue is resolved and no further action is necessary."}},
47
+ "urgency": {"type": "choice", "instructions": "Determine urgency from actual harm rather than requested urgency.",
48
+ "criteria": {"low": "Resolved or minor routine issue.", "medium": "Unresolved but contained problem.",
49
+ "high": "Severe ongoing harm or substantial financial exposure."}},
50
+ "verification": {"type": "choice", "instructions": "Determine whether the evidence is sufficient to proceed.",
51
+ "criteria": {"sufficient": "Verified identity and consistent supporting evidence.",
52
+ "insufficient": "Unverified identity, missing evidence, or conflicting facts."}},
53
+ }} for i in range(count)]
54
+
55
+
56
+ def read_records(path, seed, count):
57
+ records = json.loads(path.read_text()) if path else corpus(seed, count)
58
+ if (not isinstance(records, list) or not records or any(not isinstance(r, dict)
59
+ or "state" not in r or "questions" not in r or r.get("images") for r in records)):
60
+ raise ValueError("Expected a nonempty array of text state/questions records")
61
+ return records
62
+
63
+
64
+ def input_digest(record):
65
+ # IDs are bookkeeping, not evidence of distinct model inputs.
66
+ return records_digest([{key: record[key] for key in ("state", "questions")}])
67
+
68
+
69
+ def main():
70
+ p = argparse.ArgumentParser(description=__doc__)
71
+ p.add_argument("--model", default="Cloudflare/clef")
72
+ p.add_argument("--scope", choices=["mlp-gate-up", "mlp", "all"], default="all")
73
+ p.add_argument("--calibration-records", type=Path)
74
+ p.add_argument("--selection-records", type=Path)
75
+ p.add_argument("--heldout-records", type=Path)
76
+ p.add_argument("--train-count", type=int, default=12)
77
+ p.add_argument("--selection-count", type=int, default=6)
78
+ p.add_argument("--heldout-count", type=int, default=16)
79
+ p.add_argument("--sample-rows", type=int, default=16)
80
+ p.add_argument("--clips", type=float, nargs="+", default=[1.0, 0.99, 0.97, 0.94, 0.90])
81
+ p.add_argument("--repeat", type=int, default=30)
82
+ p.add_argument("--weights-dir", type=Path, required=True)
83
+ p.add_argument("--output", type=Path, default=Path("clef-int8-calibration.json"))
84
+ args = p.parse_args()
85
+ if min(args.repeat, args.sample_rows, args.train_count, args.selection_count, args.heldout_count) < 1:
86
+ p.error("Counts and repeats must be positive")
87
+ records = [read_records(args.calibration_records, 101, args.train_count),
88
+ read_records(args.selection_records, 211, args.selection_count),
89
+ read_records(args.heldout_records, 4242, args.heldout_count)]
90
+ hashes = [{input_digest(r) for r in group} for group in records]
91
+ if any(hashes[i] & hashes[j] for i in range(3) for j in range(i + 1, 3)):
92
+ raise ValueError("Calibration, selection, and held-out records must not overlap")
93
+ path = Path(args.model)
94
+ if not path.is_dir():
95
+ path = Path(snapshot_download(args.model, local_files_only=True))
96
+ joint = bench.load_joint_schema_module(path)
97
+ device = torch.device("cuda")
98
+ result = {"source_digest": source_digest(path), "model_path": str(path),
99
+ "torch": torch.__version__, "device": torch.cuda.get_device_name(),
100
+ "scope": args.scope, "group_size": 256,
101
+ "calibration_digest": records_digest(records[0]), "selection_digest": records_digest(records[1]),
102
+ "heldout_digest": records_digest(records[2]), "records": [len(r) for r in records],
103
+ "data_source": "external" if any([args.calibration_records, args.selection_records, args.heldout_records]) else "synthetic",
104
+ "split_sources": {split: "external" if filename else "synthetic" for split, filename in zip(
105
+ ("calibration", "selection", "heldout"),
106
+ (args.calibration_records, args.selection_records, args.heldout_records), strict=True)},
107
+ "calibration_config": {"clips": args.clips, "sample_rows_per_record": args.sample_rows,
108
+ "ridge": 0.05, "max_gain": 0.03, "min_improvement": 0.005,
109
+ "tile": [128, 256], "cluster": [2, 1]},
110
+ "method": "per-output-row clipping, bounded ridge scale/bias fit, independent local selection",
111
+ "quality_scope": "BF16 agreement, not production task accuracy", "linears": {}, "evaluation": {}}
112
+ args.output.parent.mkdir(parents=True, exist_ok=True)
113
+ def save():
114
+ args.output.write_text(json.dumps(result, indent=2) + "\n")
115
+ with torch.inference_mode():
116
+ model, processor = joint.load_release_model(path, device=device, dtype=torch.bfloat16)
117
+ def components(model):
118
+ base = model.language_model.get_base_model() if hasattr(model.language_model, "get_base_model") else model.language_model
119
+ return base, getattr(base.model, "language_model", base.model)
120
+ base, text = components(model)
121
+ restore = patch_text(text, gva=True, gates=True, heads=True, share_mlp_quant=True, share_attn_quant=True)
122
+ def encode(group):
123
+ return [joint.collate_records([joint.encode_record(processor.tokenizer, r, processor=processor, max_length=16384)],
124
+ processor.tokenizer.pad_token_id, device) for r in group]
125
+ train_batches, selection_batches, heldout_batches = [encode(group) for group in records]
126
+ regression_records = [bench.make_record(i, 8, 4, words) for i, words in enumerate([32, 256, 1024])] + helpers.semantic_records()
127
+ regression_batches = encode(regression_records)
128
+ all_eval_batches = heldout_batches + regression_batches
129
+ def backbone(batch):
130
+ return text(input_ids=batch["input_ids"], attention_mask=batch["attention_mask"], use_cache=False, return_dict=True)
131
+ def run(batch):
132
+ outputs = backbone(batch)
133
+ logits = model.head(outputs.last_hidden_state, batch["input_ids"], batch["attention_mask"],
134
+ batch["records"], base.get_output_embeddings().weight)
135
+ return outputs, logits
136
+ references = [helpers.cpu_outputs(*run(batch)) for batch in all_eval_batches]
137
+ def evaluate(label):
138
+ rows = []
139
+ for i, batch in enumerate(all_eval_batches):
140
+ hidden, logits = helpers.cpu_outputs(*run(batch))
141
+ expected_hidden, expected_logits = references[i]
142
+ rows.append({"case": i, "split": "heldout" if i < len(heldout_batches) else "regression",
143
+ "tokens": batch["input_ids"].shape[1], "hidden_error": helpers.hidden_error(hidden, expected_hidden),
144
+ "drift": decision_drift(logits, expected_logits)})
145
+ timing_batch = regression_batches[1]
146
+ for _ in range(3):
147
+ run(timing_batch)
148
+ torch.cuda.reset_peak_memory_stats()
149
+ times = [bench.timed(lambda: run(timing_batch), device)[1] for _ in range(args.repeat)]
150
+ result["evaluation"][label] = {"cases": rows, "total_ms": bench.summarize_ms(times),
151
+ "resident_vram_bytes": torch.cuda.memory_allocated(),
152
+ "peak_vram_bytes": torch.cuda.max_memory_allocated()}
153
+ print(json.dumps({"mode": label, "ms": result["evaluation"][label]["total_ms"]["median"],
154
+ "heldout_matched": sum(r["drift"]["decisions_matched"] for r in rows if r["split"] == "heldout"),
155
+ "heldout_total": sum(r["drift"]["decisions_total"] for r in rows if r["split"] == "heldout"),
156
+ "regression_matched": sum(r["drift"]["decisions_matched"] for r in rows if r["split"] == "regression")}), flush=True)
157
+ save()
158
+ evaluate("bf16")
159
+ names = projection_names(text, args.scope)
160
+ train = capture_samples(text, train_batches, backbone, names, args.sample_rows)
161
+ selection = capture_samples(text, selection_batches, backbone, names, args.sample_rows)
162
+ quantize_text(text, scope=args.scope, cluster=(2, 1))
163
+ evaluate("uncalibrated")
164
+ restore()
165
+ del restore, model, base, text, processor
166
+ gc.collect()
167
+ torch.cuda.empty_cache()
168
+ model, processor = joint.load_release_model(path, device=device, dtype=torch.bfloat16)
169
+ base, text = components(model)
170
+ restore = patch_text(text, gva=True, gates=True, heads=True, share_mlp_quant=True, share_attn_quant=True)
171
+ writer = CalibrationWriter(args.weights_dir, {key: result[key] for key in (
172
+ "source_digest", "model_path", "scope", "calibration_digest", "selection_digest", "method",
173
+ "data_source", "split_sources", "calibration_config")})
174
+ started = time.perf_counter()
175
+ for i, name in enumerate(names):
176
+ parent_name, leaf = name.rsplit(".", 1)
177
+ parent = text.get_submodule(parent_name)
178
+ linear = getattr(parent, leaf)
179
+ calibrated, stats = calibrate_linear(linear, train.pop(name), selection.pop(name), clips=args.clips)
180
+ setattr(parent, leaf, calibrated)
181
+ del linear
182
+ writer.add(name, calibrated)
183
+ result["linears"][name] = stats
184
+ if i % 16 == 0 or i == len(names) - 1:
185
+ print(json.dumps({"linear": i + 1, "total": len(names), "name": name,
186
+ "selection_mse_ratio": stats["selection_mse_ratio"]}), flush=True)
187
+ save()
188
+ result["calibration_seconds"] = time.perf_counter() - started
189
+ result["calibration_index"] = str(writer.finish())
190
+ del writer, calibrated, parent
191
+ torch.cuda.empty_cache()
192
+ evaluate("calibrated")
193
+ expected_reload = [helpers.cpu_outputs(*run(batch)) for batch in all_eval_batches]
194
+ restore()
195
+ del restore, model, base, text, processor
196
+ gc.collect()
197
+ torch.cuda.empty_cache()
198
+ model, processor = joint.load_release_model(path, device=device, dtype=torch.bfloat16)
199
+ base, text = components(model)
200
+ quantize_text(text, scope=args.scope, cluster=(2, 1), calibration_index=Path(result["calibration_index"]))
201
+ restore = patch_text(text, gva=True, gates=True, heads=True, share_mlp_quant=True, share_attn_quant=True)
202
+ try:
203
+ for batch, (expected_hidden, expected_logits) in zip(all_eval_batches, expected_reload, strict=True):
204
+ hidden, logits = helpers.cpu_outputs(*run(batch))
205
+ torch.testing.assert_close(hidden, expected_hidden, atol=0, rtol=0)
206
+ if decision_drift(logits, expected_logits)["logits_max_abs"] != 0:
207
+ raise AssertionError("Indexed calibrated checkpoint changed logits after reload")
208
+ result["indexed_reload_exact"] = True
209
+ finally:
210
+ restore()
211
+ save()
212
+
213
+
214
+ if __name__ == "__main__":
215
+ main()
runtime/clef-int8-calibration-check.py ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Check FP32 ConvRot calibration and CuTe epilogue/reload parity on H100."""
3
+
4
+ import json
5
+ from pathlib import Path
6
+ import tempfile
7
+
8
+ import torch
9
+
10
+ from clef_int8_calibration import CalibrationIndex, CalibrationWriter, calibrate_linear, integer_projection
11
+
12
+
13
+ def main():
14
+ torch.manual_seed(42)
15
+ with torch.inference_mode():
16
+ text = torch.nn.Module().cuda().eval()
17
+ text.proj = torch.nn.Linear(1024, 1024, bias=False, device="cuda", dtype=torch.bfloat16).eval()
18
+ def samples(rows):
19
+ x = torch.randn(rows, 1024, device="cuda", dtype=torch.bfloat16)
20
+ return x.cpu(), text.proj(x).cpu()
21
+ module, stats = calibrate_linear(text.proj, samples(128), samples(64))
22
+ assert stats["selection_mse_after"] <= stats["selection_mse_before"]
23
+ assert module.bias.count_nonzero() > 0
24
+ x = torch.randn(33, 1024, device="cuda", dtype=torch.bfloat16)
25
+ q, s = module.quantize_input(x)
26
+ expected = (integer_projection(q, s, module.weight) * module.weight_scale + module.bias.float()).to(torch.bfloat16)
27
+ torch.testing.assert_close(module(x), expected, rtol=0, atol=0)
28
+ with tempfile.TemporaryDirectory() as tmp:
29
+ writer = CalibrationWriter(Path(tmp), {"scope": "all"})
30
+ writer.add("proj", module)
31
+ index = CalibrationIndex(writer.finish(), text, "all")
32
+ loaded = index.load("proj", text.proj, backend="cute", tile=(128, 256), cluster=(2, 1))
33
+ torch.testing.assert_close(loaded(x), expected, rtol=0, atol=0)
34
+ stream = torch.cuda.Stream()
35
+ stream.wait_stream(torch.cuda.current_stream())
36
+ with torch.cuda.stream(stream):
37
+ streamed = loaded(x)
38
+ torch.cuda.current_stream().wait_stream(stream)
39
+ torch.testing.assert_close(streamed, expected, rtol=0, atol=0)
40
+ static = x.clone()
41
+ for _ in range(3):
42
+ loaded(static)
43
+ torch.cuda.synchronize()
44
+ graph = torch.cuda.CUDAGraph()
45
+ with torch.cuda.graph(graph):
46
+ captured = loaded(static)
47
+ changed = torch.randn_like(x)
48
+ static.copy_(changed)
49
+ graph.replay()
50
+ torch.testing.assert_close(captured, loaded(changed), rtol=0, atol=0)
51
+ print(json.dumps({"fp32_rotation": True, "cute_epilogue_exact": True, "reload_exact": True,
52
+ "nondefault_stream_exact": True, "changed_input_graph_exact": True, **stats}))
53
+
54
+
55
+ if __name__ == "__main__":
56
+ main()
runtime/clef_int8_calibration.py ADDED
@@ -0,0 +1,275 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Activation-reconstruction calibration for the existing CLEF W8A8 runtime."""
2
+
3
+ import hashlib
4
+ import json
5
+ from pathlib import Path
6
+
7
+ import torch
8
+ from safetensors import safe_open
9
+ from safetensors.torch import save_file
10
+
11
+ from clef_text_int8 import ConvRotLinear
12
+
13
+
14
+ def projection_names(text, scope):
15
+ if scope not in {"mlp-gate-up", "mlp", "all"}:
16
+ raise ValueError("Unknown calibration scope")
17
+ return [name for name, module in text.named_modules()
18
+ if isinstance(module, torch.nn.Linear) and min(module.in_features, module.out_features) >= 1024
19
+ and (scope == "all" or ".mlp." in name)
20
+ and (scope != "mlp-gate-up" or name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj"})]
21
+
22
+
23
+ def records_digest(records):
24
+ return hashlib.sha256(json.dumps(records, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
25
+
26
+
27
+ def source_digest(path):
28
+ digest = hashlib.sha256()
29
+ for name in ("config.json", "model.safetensors.index.json", "joint_schema_model.py"):
30
+ digest.update(name.encode())
31
+ digest.update((path / name).read_bytes())
32
+ return digest.hexdigest()
33
+
34
+
35
+ def capture_samples(text, batches, run, names, rows_per_record=16):
36
+ """Bounded BF16 CPU samples, not full intermediate activations or a state dict."""
37
+ if rows_per_record < 1:
38
+ raise ValueError("Rows per record must be positive")
39
+ collected = {name: ([], []) for name in names}
40
+ handles = []
41
+ for name in names:
42
+ def hook(module, inputs, output, name=name):
43
+ x = inputs[0].reshape(-1, module.in_features)
44
+ y = output.reshape(-1, module.out_features)
45
+ count = min(rows_per_record, x.shape[0])
46
+ indices = torch.linspace(0, x.shape[0] - 1, count, device=x.device).long()
47
+ collected[name][0].append(x.index_select(0, indices).cpu())
48
+ collected[name][1].append(y.index_select(0, indices).cpu())
49
+ handles.append(text.get_submodule(name).register_forward_hook(hook))
50
+ try:
51
+ for batch in batches:
52
+ run(batch)
53
+ finally:
54
+ for handle in handles:
55
+ handle.remove()
56
+ return {name: (torch.cat(x), torch.cat(y)) for name, (x, y) in collected.items()}
57
+
58
+
59
+ def fit_affine(z, target, *, affine, ridge=0.05, max_gain=0.03):
60
+ """Fit a bounded gain, shrunk towards one; optionally fit a shrunk offset."""
61
+ if z.shape != target.shape or z.ndim != 2 or not 0 <= max_gain < 1 or ridge < 0:
62
+ raise ValueError("Invalid calibration regression")
63
+ power = z.square().mean(0)
64
+ cross = (z * target).mean(0)
65
+ mean_z, mean_y = z.mean(0), target.mean(0)
66
+ if affine:
67
+ cross = cross - mean_z * mean_y
68
+ variance = (power - mean_z.square()).clamp_min(0)
69
+ else:
70
+ variance = power
71
+ gain = (cross + ridge * power) / (variance + ridge * power).clamp_min(1e-20)
72
+ gain = torch.where(power > 1e-20, gain, torch.ones_like(gain)).clamp(1 - max_gain, 1 + max_gain)
73
+ offset = (mean_y - mean_z * gain) / (1 + ridge) if affine else torch.zeros_like(gain)
74
+ return gain, offset
75
+
76
+
77
+ def integer_projection(qx, xs, qw):
78
+ padding = max(24, (qx.shape[0] + 7) // 8 * 8) - qx.shape[0]
79
+ padded = torch.nn.functional.pad(qx, (0, 0, 0, padding)) if padding else qx
80
+ return torch._int_mm(padded, qw.t())[:qx.shape[0]].float() * xs
81
+
82
+
83
+ def calibrate_linear(linear, train, selection, *, clips=(1.0, 0.99, 0.97, 0.94, 0.90),
84
+ block_rows=512, min_improvement=0.005, ridge=0.05, max_gain=0.03,
85
+ tile=(128, 256), cluster=(2, 1)):
86
+ from comfy_kitchen import _cuda_backend as cuda
87
+
88
+ if torch.is_grad_enabled() or linear.training or block_rows < 8 or block_rows % 8:
89
+ raise ValueError("Calibrate evaluated weights in inference mode with aligned row blocks")
90
+ if not clips or any(not 0 < c <= 1 for c in clips) or not 0 <= min_improvement < 1:
91
+ raise ValueError("Invalid clipping or improvement threshold")
92
+ module = ConvRotLinear(linear, tile=tile, cluster=cluster)
93
+ for samples in (train, selection):
94
+ if (len(samples) != 2 or samples[0].ndim != 2 or samples[1].ndim != 2
95
+ or samples[0].shape[0] != samples[1].shape[0] or not samples[0].shape[0]
96
+ or samples[0].shape[1] != linear.in_features or samples[1].shape[1] != linear.out_features):
97
+ raise ValueError("Invalid calibration sample shapes")
98
+ tx, ty = [v.to(linear.weight.device) for v in train]
99
+ vx, vy = [v.to(linear.weight.device) for v in selection]
100
+ tq, ts = module.quantize_input(tx)
101
+ vq, vs = module.quantize_input(vx)
102
+ ty, vy = ty.float(), vy.float()
103
+ if not all(torch.isfinite(v).all() for v in (ty, vy, tx, vx)):
104
+ raise ValueError("Non-finite calibration samples")
105
+ # The runtime's fused quantizer rotates in FP32 before INT8 rounding.
106
+ rotated = cuda.rotate_int8_convrot_weight(linear.weight.detach().float().contiguous(), 256)
107
+ selected, before, after = [], [], []
108
+ for start in range(0, module.out_features, block_rows):
109
+ stop = min(module.out_features, start + block_rows)
110
+ original_q = module.weight[start:stop].clone()
111
+ original_s = module.weight_scale[start:stop].clone()
112
+ original_b = module.bias[start:stop].clone()
113
+ target_t, target_v = ty[:, start:stop], vy[:, start:stop]
114
+ baseline = (integer_projection(vq, vs, original_q) * original_s + original_b.float()).to(torch.bfloat16).float()
115
+ best_loss = (baseline - target_v).square().mean(0)
116
+ before.append(best_loss.clone())
117
+ best_q, best_s, best_b = original_q.clone(), original_s.clone(), original_b.clone()
118
+ labels = torch.zeros(stop - start, device=linear.weight.device, dtype=torch.int32)
119
+ w = rotated[start:stop]
120
+ absmax = w.abs().amax(1).clamp_min(1e-20)
121
+ candidates = [(original_q, original_s)]
122
+ for clip in clips:
123
+ scale = absmax * (clip / 127)
124
+ q = (w / scale[:, None]).round().clamp(-127, 127).to(torch.int8)
125
+ candidates.append((q, scale))
126
+ for candidate_index, (q, prior_s) in enumerate(candidates):
127
+ u_train = integer_projection(tq, ts, q)
128
+ u_val = integer_projection(vq, vs, q)
129
+ for affine in (False, True):
130
+ gain, offset = fit_affine(u_train * prior_s, target_t - original_b.float(),
131
+ affine=affine, ridge=ridge, max_gain=max_gain)
132
+ scale = (prior_s * gain).contiguous()
133
+ bias = (original_b.float() + offset).to(torch.bfloat16)
134
+ predicted = (u_val * scale + bias.float()).to(torch.bfloat16).float()
135
+ loss = (predicted - target_v).square().mean(0)
136
+ better = torch.isfinite(loss) & (loss < best_loss * (1 - min_improvement))
137
+ best_loss = torch.where(better, loss, best_loss)
138
+ best_q = torch.where(better[:, None], q, best_q)
139
+ best_s = torch.where(better, scale, best_s)
140
+ best_b = torch.where(better, bias, best_b)
141
+ labels = torch.where(better, 1 + candidate_index * 2 + int(affine), labels)
142
+ module.weight[start:stop].copy_(best_q)
143
+ module.weight_scale[start:stop].copy_(best_s)
144
+ module.bias[start:stop].copy_(best_b)
145
+ selected.append(labels.cpu())
146
+ after.append(best_loss)
147
+ before, after = torch.cat(before), torch.cat(after)
148
+ labels = torch.cat(selected)
149
+ stats = {"train_rows": tx.shape[0], "selection_rows": vx.shape[0], "output_rows": module.out_features,
150
+ "changed_rows": int((labels != 0).sum()), "clipping": list(clips),
151
+ "selection_mse_before": float(before.mean()), "selection_mse_after": float(after.mean()),
152
+ "selection_mse_ratio": float(torch.where(before.sum() > 0, after.sum() / before.sum().clamp_min(1e-20), 1)),
153
+ "choices": torch.bincount(labels, minlength=1 + len(candidates) * 2).tolist()}
154
+ return module, stats
155
+
156
+
157
+ class CalibrationWriter:
158
+ """Write bounded, tensor-preserving safetensors shards without a full state dict."""
159
+
160
+ def __init__(self, directory, metadata, max_bytes=1_900_000_000):
161
+ if type(max_bytes) is not int or max_bytes < 1:
162
+ raise ValueError("Shard size must be a positive integer")
163
+ self.directory = Path(directory)
164
+ self.directory.mkdir(parents=True, exist_ok=True)
165
+ if any(self.directory.iterdir()):
166
+ raise ValueError("Calibration output directory must be empty")
167
+ self.metadata, self.max_bytes = metadata, max_bytes
168
+ self.pending, self.pending_bytes, self.total_bytes = {}, 0, 0
169
+ self.shards, self.weight_map = [], {}
170
+ self.stem = "clef_text_int8_convrot_calibrated"
171
+
172
+ def add(self, name, module):
173
+ for leaf in ("weight", "weight_scale", "bias"):
174
+ key, tensor = name + "." + leaf, getattr(module, leaf)
175
+ size = tensor.numel() * tensor.element_size()
176
+ if key in self.weight_map or key in self.pending:
177
+ raise ValueError("Duplicate calibrated tensor")
178
+ if self.pending and self.pending_bytes + size > self.max_bytes:
179
+ self.flush()
180
+ self.pending[key] = tensor.detach().cpu().contiguous()
181
+ self.pending_bytes += size
182
+ self.total_bytes += size
183
+
184
+ def flush(self):
185
+ if not self.pending:
186
+ return
187
+ name = f"{self.stem}-{len(self.shards) + 1:05d}.safetensors"
188
+ save_file(self.pending, self.directory / name, metadata={"format": "clef-convrot-linear-overrides-v1"})
189
+ self.weight_map.update({key: name for key in self.pending})
190
+ self.shards.append(name)
191
+ self.pending, self.pending_bytes = {}, 0
192
+
193
+ def finish(self):
194
+ self.flush()
195
+ if not self.shards:
196
+ raise ValueError("No calibrated weights")
197
+ renamed = {}
198
+ for i, old in enumerate(self.shards, 1):
199
+ new = f"{self.stem}-{i:05d}-of-{len(self.shards):05d}.safetensors"
200
+ (self.directory / old).rename(self.directory / new)
201
+ renamed[old] = new
202
+ path = self.directory / (self.stem + ".safetensors.index.json")
203
+ path.write_text(json.dumps({"metadata": {**self.metadata, "total_size": self.total_bytes,
204
+ "format": "clef-convrot-linear-overrides-v1", "group_size": 256},
205
+ "weight_map": {k: renamed[v] for k, v in self.weight_map.items()}}, indent=2) + "\n")
206
+ return path
207
+
208
+
209
+ def unique_object(pairs):
210
+ result = {}
211
+ for key, value in pairs:
212
+ if key in result:
213
+ raise ValueError("Duplicate calibration index key")
214
+ result[key] = value
215
+ return result
216
+
217
+
218
+ class CalibrationIndex:
219
+ def __init__(self, path, text, scope):
220
+ self.path = Path(path)
221
+ index = json.loads(self.path.read_text(), object_pairs_hook=unique_object)
222
+ if (not isinstance(index, dict) or not isinstance(index.get("metadata"), dict)
223
+ or not isinstance(index.get("weight_map"), dict)):
224
+ raise ValueError("Malformed calibration index")
225
+ self.metadata, self.weight_map = index["metadata"], index["weight_map"]
226
+ if (self.metadata.get("format") != "clef-convrot-linear-overrides-v1"
227
+ or self.metadata.get("group_size") != 256 or self.metadata.get("scope") != scope):
228
+ raise ValueError("Calibration format/group/scope mismatch")
229
+ self.names = projection_names(text, scope)
230
+ expected = {name + "." + leaf for name in self.names for leaf in ("weight", "weight_scale", "bias")}
231
+ if set(self.weight_map) != expected:
232
+ raise ValueError("Incomplete or unexpected calibrated tensor coverage")
233
+ if any(not isinstance(filename, str) for filename in self.weight_map.values()):
234
+ raise ValueError("Unsafe calibration shard reference")
235
+ self.files, total = {}, 0
236
+ for filename in sorted(set(self.weight_map.values())):
237
+ if (not isinstance(filename, str) or not filename or "/" in filename or "\\" in filename
238
+ or filename in {".", ".."} or ":" in filename):
239
+ raise ValueError("Unsafe calibration shard reference")
240
+ resolved = (self.path.parent / filename).resolve()
241
+ if not resolved.is_relative_to(self.path.parent.resolve()) or not resolved.is_file():
242
+ raise ValueError("Missing or unsafe calibration shard")
243
+ with safe_open(resolved, framework="pt", device="cpu") as f:
244
+ assigned = {key for key, value in self.weight_map.items() if value == filename}
245
+ if set(f.keys()) != assigned:
246
+ raise ValueError("Calibration shard/index tensor mismatch")
247
+ for key in assigned:
248
+ name, leaf = key.rsplit(".", 1)
249
+ linear = text.get_submodule(name)
250
+ shape, dtype, size = ((linear.out_features, linear.in_features), "I8", 1) if leaf == "weight" else (
251
+ ((linear.out_features,), "F32", 4) if leaf == "weight_scale" else ((linear.out_features,), "BF16", 2))
252
+ view = f.get_slice(key)
253
+ if tuple(view.get_shape()) != shape or view.get_dtype() != dtype:
254
+ raise ValueError("Calibrated tensor shape/dtype mismatch")
255
+ total += size * torch.Size(shape).numel()
256
+ self.files[filename] = resolved
257
+ if type(self.metadata.get("total_size")) is not int or total != self.metadata["total_size"]:
258
+ raise ValueError("Calibration total_size mismatch")
259
+
260
+ def load(self, name, linear, *, backend, tile, cluster):
261
+ tensors = []
262
+ for leaf in ("weight", "weight_scale", "bias"):
263
+ key = name + "." + leaf
264
+ with safe_open(self.files[self.weight_map[key]], framework="pt", device=str(linear.weight.device)) as f:
265
+ tensor = f.get_tensor(key)
266
+ if not torch.isfinite(tensor).all() or (leaf == "weight_scale" and (tensor <= 0).any()):
267
+ raise ValueError("Non-finite calibrated tensor or nonpositive scale")
268
+ tensors.append(tensor)
269
+ module = ConvRotLinear.__new__(ConvRotLinear)
270
+ torch.nn.Module.__init__(module)
271
+ module.in_features, module.out_features = linear.in_features, linear.out_features
272
+ module.group_size, module.backend, module.tile, module.cluster = 256, backend, tile, cluster
273
+ for leaf, tensor in zip(("weight", "weight_scale", "bias"), tensors, strict=True):
274
+ module.register_buffer(leaf, tensor)
275
+ return module.eval()
runtime/clef_int8_cutlass.py ADDED
@@ -0,0 +1,1307 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: BSD-3-Clause
3
+
4
+ # Redistribution and use in source and binary forms, with or without
5
+ # modification, are permitted provided that the following conditions are met:
6
+
7
+ # 1. Redistributions of source code must retain the above copyright notice, this
8
+ # list of conditions and the following disclaimer.
9
+
10
+ # 2. Redistributions in binary form must reproduce the above copyright notice,
11
+ # this list of conditions and the following disclaimer in the documentation
12
+ # and/or other materials provided with the distribution.
13
+
14
+ # 3. Neither the name of the copyright holder nor the names of its
15
+ # contributors may be used to endorse or promote products derived from
16
+ # this software without specific prior written permission.
17
+
18
+ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19
+ # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20
+ # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21
+ # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22
+ # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23
+ # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24
+ # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25
+ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26
+ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27
+ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
+
29
+ from typing import Type
30
+ import math
31
+ import cuda.bindings.driver as cuda
32
+
33
+ import cutlass
34
+ import cutlass.cute as cute
35
+ import cutlass.utils as utils
36
+ import cutlass.pipeline as pipeline
37
+ from cutlass.pipeline import pipeline_init_arrive, pipeline_init_wait
38
+ import cutlass.utils.hopper_helpers as sm90_utils
39
+ from clef_vision_norm import mul_rn
40
+
41
+ # Adapted from CUTLASS 4.7.1 examples/python/CuTeDSL/hopper/dense_gemm.py.
42
+
43
+ """
44
+ A high-performance batched dense GEMM (C = A * B) example for the NVIDIA Hopper architecture
45
+ using CuTe DSL.
46
+ - Matrix A is MxKxL, L is batch dimension, A can be row-major("K") or column-major("M")
47
+ - Matrix B is NxKxL, L is batch dimension, B can be row-major("N") or column-major("K")
48
+ - Matrix C is MxNxL, L is batch dimension, C can be row-major("N") or column-major("M")
49
+
50
+ This GEMM kernel supports the following features:
51
+ - Utilizes Tensor Memory Access (TMA) for efficient memory operations
52
+ - Utilizes Hopper's WGMMA for matrix multiply-accumulate (MMA) operations
53
+ - Implements TMA multicast with cluster to reduce L2 memory traffic
54
+ - Supports multi-stage pipeline to overlap computation and memory access
55
+
56
+ This GEMM works as follows:
57
+ 1. Load A and B matrices from global memory (GMEM) to shared memory (SMEM) using TMA operations.
58
+ 2. Perform matrix multiply-accumulate (MMA) operations using WGMMA instruction.
59
+ 3. Store results from registers (RMEM) to shared memory (SMEM), then to global memory (GMEM) with TMA operations.
60
+
61
+ Hopper WGMMA instructions operate as follows:
62
+ - Read matrix A from SMEM
63
+ - Read matrix B from SMEM
64
+ - Perform MMA operation and store the result in Accumulator(register)
65
+
66
+ To run this example:
67
+
68
+ .. code-block:: bash
69
+
70
+ python examples/hopper/dense_gemm.py \
71
+ --mnkl 8192,8192,8192,1 --tile_shape_mn 128,256 \
72
+ --cluster_shape_mn 1,1 --a_dtype Float16 --b_dtype Float16 \
73
+ --c_dtype Float16 --acc_dtype Float32 \
74
+ --a_major k --b_major k --c_major n
75
+
76
+ The above example command compute batched gemm with M=8192, N=8192, K=8192,
77
+ batch_count=1. The Hopper WGMMA tile shape is 128x256x64 and the cluster shape
78
+ is (1,1). The input, mma accumulator and output data type are set as fp16, fp32
79
+ and fp16, respectively.
80
+
81
+ To collect performance with NCU profiler:
82
+
83
+ .. code-block:: bash
84
+
85
+ ncu python examples/hopper/dense_gemm.py \
86
+ --mnkl 8192,8192,8192,1 --tile_shape_mn 128,256 \
87
+ --cluster_shape_mn 1,1 --a_dtype Float16 --b_dtype Float16 \
88
+ --c_dtype Float16 --acc_dtype Float32 \
89
+ --a_major k --b_major k --c_major n
90
+
91
+ Constraints:
92
+ * Supported input data types: fp16, fp8 (e4m3fn, e5m2), int8, uint8
93
+ * For fp16 types, A and B must have the same data type
94
+ * For fp8 types, A and B can have different types (e4m3fn or e5m2)
95
+ * For 8-bit integer types, A and B can have different types (int8 or uint8)
96
+ * 8-bit types (e4m3fn, e5m2, int8, uint8) only support k-major layout
97
+ * CTA tile shape M must be 64/128
98
+ * CTA tile shape N must be 64/128/256
99
+ * Cluster shape M/N must be positive and power of 2, total cluster size <= 4
100
+ * The contiguous dimension of A/B/C tensors must be at least 16 bytes aligned,
101
+ i.e, number of elements is a multiple of 8, 16 for Float16, and Float8, respectively.
102
+ """
103
+
104
+
105
+ class HopperWgmmaGemmKernel:
106
+ """
107
+ This class implements batched matrix multiplication (C = A x B) with support for various data types
108
+ and architectural features specific to Hopper GPUs.
109
+
110
+ :param acc_dtype: Data type for accumulation during computation
111
+ :type acc_dtype: type[cutlass.Numeric]
112
+ :param tile_shape_mn: Shape of the CTA tile (M,N)
113
+ :type tile_shape_mn: Tuple[int, int]
114
+ :param cluster_shape_mn: Cluster dimensions (M,N) for parallel processing
115
+ :type cluster_shape_mn: Tuple[int, int]
116
+
117
+ :note: Supported A/B data types:
118
+ - Float16
119
+ A and B must have the same data type
120
+ - Float8E4M3FN/Float8E5M2
121
+ A and B can have different types (Float8E4M3FN/Float8E5M2)
122
+ only support k-major layout
123
+ - Int8/Uint8
124
+ A and B can have different types (Int8/Uint8)
125
+ only support k-major layout
126
+
127
+ :note: Supported accumulation types:
128
+ - Float32/Float16 (for all floating point inputs)
129
+ - Int32 (for Int8/Uint8 inputs)
130
+
131
+ :note: Constraints:
132
+ - CTA tile M must be 64/128
133
+ - CTA tile N must be 64/128/256
134
+ - CTA tile K must be 64
135
+ - Cluster shape M/N must be positive and power of 2, total cluster size <= 4
136
+
137
+ Example:
138
+ >>> gemm = HopperWgmmaGemmKernel(
139
+ ... acc_dtype=cutlass.Float32,
140
+ ... tile_shape_mn=(128, 256),
141
+ ... cluster_shape_mn=(1, 1)
142
+ ... )
143
+ >>> gemm(a_tensor, b_tensor, c_tensor, stream)
144
+ """
145
+
146
+ def __init__(
147
+ self,
148
+ acc_dtype: type[cutlass.Numeric],
149
+ tile_shape_mn: tuple[int, int],
150
+ cluster_shape_mn: tuple[int, int],
151
+ ):
152
+ """
153
+ Initializes the configuration for a Hopper dense GEMM kernel.
154
+
155
+ This configuration includes data types for operands, tile shape, cluster configuration,
156
+ and thread layout.
157
+
158
+ :param acc_dtype: Data type for accumulation during computation
159
+ :type acc_dtype: type[cutlass.Numeric]
160
+ :param tile_shape_mn: Shape of the CTA tile (M,N)
161
+ :type tile_shape_mn: Tuple[int, int]
162
+ :param cluster_shape_mn: Cluster dimensions (M,N) for parallel processing
163
+ :type cluster_shape_mn: Tuple[int, int]
164
+ """
165
+
166
+ self.acc_dtype = acc_dtype
167
+
168
+ self.cluster_shape_mn = cluster_shape_mn
169
+ self.mma_inst_shape_mn = None
170
+ # K dimension is deferred in _setup_attributes
171
+ self.tile_shape_mnk = (*tile_shape_mn, 1)
172
+ # For large tile size, using two warp groups is preferred because using only one warp
173
+ # group may result in register spill
174
+ self.atom_layout_mnk = (
175
+ (2, 1, 1)
176
+ if self.tile_shape_mnk[0] > 64
177
+ else (1, 1, 1)
178
+ )
179
+ self.num_mcast_ctas_a = None
180
+ self.num_mcast_ctas_b = None
181
+ self.is_a_mcast = False
182
+ self.is_b_mcast = False
183
+ self.tiled_mma = None
184
+
185
+ self.occupancy = 1
186
+ self.mma_warp_groups = math.prod(self.atom_layout_mnk)
187
+ self.num_threads_per_warp_group = 128
188
+ self.threads_per_cta = self.mma_warp_groups * self.num_threads_per_warp_group
189
+ self.smem_capacity = utils.get_smem_capacity_in_bytes("sm_90")
190
+
191
+ self.ab_stage = None
192
+ self.epi_stage = None
193
+
194
+ self.a_smem_layout_staged = None
195
+ self.b_smem_layout_staged = None
196
+ self.epi_smem_layout_staged = None
197
+ self.epi_tile = None
198
+
199
+ self.shared_storage = None
200
+ self.buffer_align_bytes = 1024
201
+
202
+ def _setup_attributes(self):
203
+ """Set up configurations that are dependent on GEMM inputs
204
+
205
+ This method configures various attributes based on the input tensor properties
206
+ (data types, leading dimensions) and kernel settings:
207
+ - Configuring tiled MMA
208
+ - Computing MMA/cluster/tile shapes
209
+ - Computing cluster layout
210
+ - Computing multicast CTAs for A/B
211
+ - Computing epilogue subtile
212
+ - Setting up A/B/C stage counts in shared memory
213
+ - Computing A/B/C shared memory layout
214
+ """
215
+
216
+ # check the cta tile shape
217
+ if self.tile_shape_mnk[0] not in [64, 128]:
218
+ raise ValueError("CTA tile shape M must be 64/128")
219
+ if self.tile_shape_mnk[1] not in [64, 128, 256]:
220
+ raise ValueError("CTA tile shape N must be 64/128/256")
221
+
222
+ self.tiled_mma = sm90_utils.make_trivial_tiled_mma(
223
+ self.a_dtype,
224
+ self.b_dtype,
225
+ self.a_layout.sm90_mma_major_mode(),
226
+ self.b_layout.sm90_mma_major_mode(),
227
+ self.acc_dtype,
228
+ self.atom_layout_mnk,
229
+ tiler_mn=(64, self.tile_shape_mnk[1]),
230
+ )
231
+ mma_inst_shape_k = cute.size(self.tiled_mma.shape_mnk, mode=[2])
232
+ mma_inst_tile_k = 4
233
+ self.tile_shape_mnk = (
234
+ self.tile_shape_mnk[0],
235
+ self.tile_shape_mnk[1],
236
+ mma_inst_shape_k * mma_inst_tile_k,
237
+ )
238
+
239
+ self.cta_layout_mnk = cute.make_layout((*self.cluster_shape_mn, 1))
240
+ self.num_mcast_ctas_a = self.cluster_shape_mn[1]
241
+ self.num_mcast_ctas_b = self.cluster_shape_mn[0]
242
+ self.is_a_mcast = self.num_mcast_ctas_a > 1
243
+ self.is_b_mcast = self.num_mcast_ctas_b > 1
244
+
245
+ is_cooperative = self.atom_layout_mnk == (2, 1, 1)
246
+ self.epi_tile = sm90_utils.compute_tile_shape_or_override(
247
+ self.tile_shape_mnk, self.c_dtype, is_cooperative=is_cooperative
248
+ )
249
+
250
+ # Compute stage before compute smem layout
251
+ self.ab_stage, self.epi_stage = self._compute_stages(
252
+ self.tile_shape_mnk,
253
+ self.a_dtype,
254
+ self.b_dtype,
255
+ self.smem_capacity,
256
+ self.occupancy,
257
+ )
258
+
259
+ (
260
+ self.a_smem_layout_staged,
261
+ self.b_smem_layout_staged,
262
+ self.epi_smem_layout_staged,
263
+ ) = self._make_smem_layouts(
264
+ self.tile_shape_mnk,
265
+ self.epi_tile,
266
+ self.a_dtype,
267
+ self.a_layout,
268
+ self.b_dtype,
269
+ self.b_layout,
270
+ self.ab_stage,
271
+ self.c_dtype,
272
+ self.c_layout,
273
+ self.epi_stage,
274
+ )
275
+
276
+ @cute.jit
277
+ def __call__(
278
+ self,
279
+ a: cute.Tensor,
280
+ b: cute.Tensor,
281
+ c: cute.Tensor,
282
+ scale_a: cute.Tensor,
283
+ scale_b: cute.Tensor,
284
+ bias: cute.Tensor,
285
+ stream: cuda.CUstream,
286
+ ):
287
+ """Execute the GEMM operation in steps:
288
+ - Setup static attributes
289
+ - Setup TMA load/store atoms and tensors
290
+ - Compute grid size
291
+ - Define shared storage for kernel
292
+ - Launch the kernel synchronously
293
+
294
+ :param a: Input tensor A
295
+ :type a: cute.Tensor
296
+ :param b: Input tensor B
297
+ :type b: cute.Tensor
298
+ :param c: Output tensor C
299
+ :type c: cute.Tensor
300
+ :param stream: CUDA stream for asynchronous execution
301
+ :type stream: cuda.CUstream
302
+ """
303
+
304
+ # setup static attributes before smem/grid/tma computation
305
+ self.a_dtype = a.element_type
306
+ self.b_dtype = b.element_type
307
+ self.c_dtype = c.element_type
308
+ self.a_layout = utils.LayoutEnum.from_tensor(a)
309
+ self.b_layout = utils.LayoutEnum.from_tensor(b)
310
+ self.c_layout = utils.LayoutEnum.from_tensor(c)
311
+
312
+ if cutlass.const_expr(
313
+ self.a_dtype.width == 16 and self.a_dtype != self.b_dtype
314
+ ):
315
+ raise TypeError(f"Type mismatch: {self.a_dtype} != {self.b_dtype}")
316
+ if cutlass.const_expr(self.a_dtype.width != self.b_dtype.width):
317
+ raise TypeError(
318
+ f"Type width mismatch: {self.a_dtype.width} != {self.b_dtype.width}"
319
+ )
320
+ if cutlass.const_expr(self.a_dtype.width != 16 and self.a_dtype.width != 8):
321
+ raise TypeError("a_dtype should be float16 or float8")
322
+
323
+ self._setup_attributes()
324
+
325
+ tma_atom_a, tma_tensor_a = self._make_tma_atoms_and_tensors(
326
+ a,
327
+ self.a_smem_layout_staged,
328
+ (self.tile_shape_mnk[0], self.tile_shape_mnk[2]),
329
+ self.cluster_shape_mn[1],
330
+ )
331
+
332
+ tma_atom_b, tma_tensor_b = self._make_tma_atoms_and_tensors(
333
+ b,
334
+ self.b_smem_layout_staged,
335
+ (self.tile_shape_mnk[1], self.tile_shape_mnk[2]),
336
+ self.cluster_shape_mn[0],
337
+ )
338
+
339
+ tma_atom_c, tma_tensor_c = self._make_tma_store_atoms_and_tensors(
340
+ c,
341
+ self.epi_smem_layout_staged,
342
+ self.epi_tile,
343
+ )
344
+
345
+ grid = self._compute_grid(c, self.tile_shape_mnk, self.cluster_shape_mn)
346
+
347
+ @cute.struct
348
+ class SharedStorage:
349
+ mainloop_pipeline_array_ptr: cute.struct.MemRange[
350
+ cutlass.Int64, self.ab_stage * 2
351
+ ]
352
+ sA: cute.struct.Align[
353
+ cute.struct.MemRange[
354
+ self.a_dtype, cute.cosize(self.a_smem_layout_staged)
355
+ ],
356
+ self.buffer_align_bytes,
357
+ ]
358
+ sB: cute.struct.Align[
359
+ cute.struct.MemRange[
360
+ self.b_dtype, cute.cosize(self.b_smem_layout_staged)
361
+ ],
362
+ self.buffer_align_bytes,
363
+ ]
364
+
365
+ self.shared_storage = SharedStorage
366
+
367
+ # Launch the kernel synchronously
368
+ self.kernel(
369
+ tma_atom_a,
370
+ tma_tensor_a,
371
+ tma_atom_b,
372
+ tma_tensor_b,
373
+ tma_atom_c,
374
+ tma_tensor_c,
375
+ scale_a,
376
+ scale_b,
377
+ bias,
378
+ self.tiled_mma,
379
+ self.cta_layout_mnk,
380
+ self.a_smem_layout_staged,
381
+ self.b_smem_layout_staged,
382
+ self.epi_smem_layout_staged,
383
+ ).launch(
384
+ grid=grid,
385
+ block=[self.threads_per_cta, 1, 1],
386
+ cluster=(*self.cluster_shape_mn, 1),
387
+ stream=stream,
388
+ )
389
+ return
390
+
391
+ # GPU device kernel
392
+ @cute.kernel
393
+ def kernel(
394
+ self,
395
+ tma_atom_a: cute.CopyAtom,
396
+ mA_mkl: cute.Tensor,
397
+ tma_atom_b: cute.CopyAtom,
398
+ mB_nkl: cute.Tensor,
399
+ tma_atom_c: cute.CopyAtom,
400
+ mC_mnl: cute.Tensor,
401
+ mScaleA: cute.Tensor,
402
+ mScaleB: cute.Tensor,
403
+ mBias: cute.Tensor,
404
+ tiled_mma: cute.TiledMma,
405
+ cta_layout_mnk: cute.Layout,
406
+ a_smem_layout_staged: cute.ComposedLayout,
407
+ b_smem_layout_staged: cute.ComposedLayout,
408
+ epi_smem_layout_staged: cute.ComposedLayout,
409
+ ):
410
+ """
411
+ GPU device kernel performing the batched GEMM computation.
412
+
413
+ :param tma_atom_a: TMA copy atom for A tensor
414
+ :type tma_atom_a: cute.CopyAtom
415
+ :param mA_mkl: Input tensor A
416
+ :type mA_mkl: cute.Tensor
417
+ :param tma_atom_b: TMA copy atom for B tensor
418
+ :type tma_atom_b: cute.CopyAtom
419
+ :param mB_nkl: Input tensor B
420
+ :type mB_nkl: cute.Tensor
421
+ :param tma_atom_c: TMA copy atom for C tensor
422
+ :type tma_atom_c: cute.CopyAtom
423
+ :param mC_mnl: Output tensor C
424
+ :type mC_mnl: cute.Tensor
425
+ :param tiled_mma: Tiled MMA object
426
+ :type tiled_mma: cute.TiledMma
427
+ :param cta_layout_mnk: CTA layout
428
+ :type cta_layout_mnk: cute.Layout
429
+ :param a_smem_layout_staged: Shared memory layout for A
430
+ :type a_smem_layout_staged: cute.ComposedLayout
431
+ :param b_smem_layout_staged: Shared memory layout for B
432
+ :type b_smem_layout_staged: cute.ComposedLayout
433
+ :param epi_smem_layout_staged: Shared memory layout for epilogue
434
+ :type epi_smem_layout_staged: cute.ComposedLayout
435
+ """
436
+
437
+ warp_idx = cute.arch.warp_idx()
438
+ warp_idx = cute.arch.make_warp_uniform(warp_idx)
439
+
440
+ # /////////////////////////////////////////////////////////////////////////////
441
+ # Prefetch Tma desc
442
+ # /////////////////////////////////////////////////////////////////////////////
443
+ if warp_idx == 0:
444
+ cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_a)
445
+ cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_b)
446
+
447
+ # ///////////////////////////////////////////////////////////////////////////////
448
+ # Get cta/warp/thread idx
449
+ # ///////////////////////////////////////////////////////////////////////////////
450
+ bidx, bidy, bidz = cute.arch.block_idx()
451
+ tidx, _, _ = cute.arch.thread_idx()
452
+
453
+ cidx, cidy, _ = cute.arch.cluster_idx()
454
+ cdimx, cdimy, _ = cute.arch.cluster_dim()
455
+ cluster_id = cidx + cdimx * cidy
456
+
457
+ # CTA Swizzle to promote L2 data reuse
458
+ group_size_m = 8
459
+ s_shape = (
460
+ (group_size_m, cdimx // group_size_m),
461
+ cdimy,
462
+ )
463
+ s_stride = ((1, cdimy * group_size_m), group_size_m)
464
+ s_layout = cute.make_layout(s_shape, stride=s_stride)
465
+ num_reg_cids = cute.size(s_shape)
466
+ cid_m, cid_n = s_layout.get_flat_coord(cluster_id % num_reg_cids)
467
+
468
+ # Deal with the tail part
469
+ if cluster_id >= num_reg_cids:
470
+ tail_size_m = cdimx % group_size_m
471
+ tail_layout = cute.make_layout(
472
+ (tail_size_m, cdimy), stride=(1, tail_size_m)
473
+ )
474
+ tail_cid = cluster_id - num_reg_cids
475
+ tail_cid_m, tail_cid_n = tail_layout.get_flat_coord(tail_cid)
476
+ cid_m = cute.size(s_shape, mode=[0]) + tail_cid_m
477
+ cid_n = tail_cid_n
478
+
479
+ # Get the pid from cluster id
480
+ bidx_in_cluster = cute.arch.block_in_cluster_idx()
481
+ pid_m = cid_m * self.cluster_shape_mn[0] + bidx_in_cluster[0]
482
+ pid_n = cid_n * self.cluster_shape_mn[1] + bidx_in_cluster[1]
483
+
484
+ tile_coord_mnkl = (pid_m, pid_n, None, bidz)
485
+ cta_rank_in_cluster = cute.arch.make_warp_uniform(
486
+ cute.arch.block_idx_in_cluster()
487
+ )
488
+ cluster_coord_mnk = cta_layout_mnk.get_flat_coord(cta_rank_in_cluster)
489
+
490
+ # ///////////////////////////////////////////////////////////////////////////////
491
+ # Get mcast mask
492
+ # ///////////////////////////////////////////////////////////////////////////////
493
+ a_mcast_mask = cute.make_layout_image_mask(
494
+ cta_layout_mnk, cluster_coord_mnk, mode=1
495
+ )
496
+ b_mcast_mask = cute.make_layout_image_mask(
497
+ cta_layout_mnk, cluster_coord_mnk, mode=0
498
+ )
499
+
500
+ a_mcast_mask = a_mcast_mask if self.is_a_mcast else 0
501
+ b_mcast_mask = b_mcast_mask if self.is_b_mcast else 0
502
+ a_smem_layout = cute.slice_(a_smem_layout_staged, (None, None, 0))
503
+ b_smem_layout = cute.slice_(b_smem_layout_staged, (None, None, 0))
504
+ tma_copy_bytes = cute.size_in_bytes(
505
+ self.a_dtype, a_smem_layout
506
+ ) + cute.size_in_bytes(self.b_dtype, b_smem_layout)
507
+
508
+ # /////////////////////////////////////////////////////////////////////////////
509
+ # Alloc and init AB full/empty + ACC full mbar (pipeline)
510
+ # /////////////////////////////////////////////////////////////////////////////
511
+ smem = cutlass.utils.SmemAllocator()
512
+ storage = smem.allocate(self.shared_storage)
513
+
514
+ # mbar arrays
515
+ mainloop_pipeline_array_ptr = storage.mainloop_pipeline_array_ptr.data_ptr()
516
+
517
+ # Threads/warps participating in this pipeline
518
+ mainloop_pipeline_producer_group = pipeline.CooperativeGroup(
519
+ pipeline.Agent.Thread
520
+ )
521
+ # Each warp will constribute to the arrive count with the number of mcast size
522
+ mcast_size = self.num_mcast_ctas_a + self.num_mcast_ctas_b - 1
523
+ num_warps = self.threads_per_cta // 32
524
+ consumer_arrive_cnt = mcast_size * num_warps
525
+ mainloop_pipeline_consumer_group = pipeline.CooperativeGroup(
526
+ pipeline.Agent.Thread, consumer_arrive_cnt
527
+ )
528
+
529
+ cta_layout_vmnk = cute.make_layout((1, *cta_layout_mnk.shape))
530
+ mainloop_pipeline = pipeline.PipelineTmaAsync.create(
531
+ barrier_storage=mainloop_pipeline_array_ptr,
532
+ num_stages=self.ab_stage,
533
+ producer_group=mainloop_pipeline_producer_group,
534
+ consumer_group=mainloop_pipeline_consumer_group,
535
+ tx_count=tma_copy_bytes,
536
+ cta_layout_vmnk=cta_layout_vmnk,
537
+ defer_sync=True,
538
+ )
539
+
540
+ # Cluster arrive after barrier init
541
+ pipeline_init_arrive(cluster_shape_mn=self.cluster_shape_mn, is_relaxed=True)
542
+
543
+ # ///////////////////////////////////////////////////////////////////////////////
544
+ # Generate smem tensor A/B
545
+ # ///////////////////////////////////////////////////////////////////////////////
546
+ sA = storage.sA.get_tensor(
547
+ a_smem_layout_staged.outer, swizzle=a_smem_layout_staged.inner
548
+ )
549
+ sB = storage.sB.get_tensor(
550
+ b_smem_layout_staged.outer, swizzle=b_smem_layout_staged.inner
551
+ )
552
+ sC_ptr = cute.recast_ptr(
553
+ sA.iterator, epi_smem_layout_staged.inner, dtype=self.c_dtype
554
+ )
555
+ sC = cute.make_tensor(sC_ptr, epi_smem_layout_staged.outer)
556
+
557
+ # ///////////////////////////////////////////////////////////////////////////////
558
+ # Local_tile partition global tensors
559
+ # ///////////////////////////////////////////////////////////////////////////////
560
+ # (bM, bK, RestK)
561
+ gA_mkl = cute.local_tile(
562
+ mA_mkl, self.tile_shape_mnk, tile_coord_mnkl, proj=(1, None, 1)
563
+ )
564
+ # (bN, bK, RestK)
565
+ gB_nkl = cute.local_tile(
566
+ mB_nkl, self.tile_shape_mnk, tile_coord_mnkl, proj=(None, 1, 1)
567
+ )
568
+ # (bM, bN)
569
+ gC_mnl = cute.local_tile(
570
+ mC_mnl, self.tile_shape_mnk, tile_coord_mnkl, proj=(1, 1, None)
571
+ )
572
+ cC_mnl = cute.make_identity_tensor(mC_mnl.shape)
573
+ gC_coord = cute.local_tile(
574
+ cC_mnl, self.tile_shape_mnk, tile_coord_mnkl, proj=(1, 1, None)
575
+ )
576
+
577
+ # //////////////////////////////////////////////////////////////////////////////
578
+ # Partition global tensor for TiledMMA_A/B/C
579
+ # //////////////////////////////////////////////////////////////////////////////
580
+ warp_group_idx = cute.arch.make_warp_uniform(
581
+ tidx // self.num_threads_per_warp_group
582
+ )
583
+ warp_group_thread_layout = cute.make_layout(
584
+ self.mma_warp_groups, stride=self.num_threads_per_warp_group
585
+ )
586
+ thr_mma = tiled_mma.get_slice(warp_group_thread_layout(warp_group_idx))
587
+
588
+ tCgC = thr_mma.partition_C(gC_mnl)
589
+
590
+ # //////////////////////////////////////////////////////////////////////////////
591
+ # Partition shared tensor for TMA load A/B
592
+ # //////////////////////////////////////////////////////////////////////////////
593
+ # TMA load A partition_S/D
594
+ a_cta_layout = cute.make_layout(cute.slice_(cta_layout_mnk, (0, None, 0)).shape)
595
+ a_cta_crd = cluster_coord_mnk[1]
596
+ sA_for_tma_partition = cute.group_modes(sA, 0, 2)
597
+ gA_for_tma_partition = cute.group_modes(gA_mkl, 0, 2)
598
+ tAsA, tAgA_mkl = cute.nvgpu.cpasync.tma_partition(
599
+ tma_atom_a,
600
+ a_cta_crd,
601
+ a_cta_layout,
602
+ sA_for_tma_partition,
603
+ gA_for_tma_partition,
604
+ )
605
+
606
+ # TMA load B partition_S/D
607
+ b_cta_layout = cute.make_layout(cute.slice_(cta_layout_mnk, (None, 0, 0)).shape)
608
+ b_cta_crd = cluster_coord_mnk[0]
609
+ sB_for_tma_partition = cute.group_modes(sB, 0, 2)
610
+ gB_for_tma_partition = cute.group_modes(gB_nkl, 0, 2)
611
+ tBsB, tBgB_nkl = cute.nvgpu.cpasync.tma_partition(
612
+ tma_atom_b,
613
+ b_cta_crd,
614
+ b_cta_layout,
615
+ sB_for_tma_partition,
616
+ gB_for_tma_partition,
617
+ )
618
+
619
+ # //////////////////////////////////////////////////////////////////////////////
620
+ # Make fragments
621
+ # //////////////////////////////////////////////////////////////////////////////
622
+ tCsA = thr_mma.partition_A(sA)
623
+ tCsB = thr_mma.partition_B(sB)
624
+ tCrA = tiled_mma.make_fragment_A(tCsA)
625
+ tCrB = tiled_mma.make_fragment_B(tCsB)
626
+
627
+ acc_shape = tCgC.shape
628
+ accumulators = cute.make_rmem_tensor(acc_shape, self.acc_dtype)
629
+
630
+ # ///////////////////////////////////////////////////////////////////////////////
631
+ # Cluster wait
632
+ # ///////////////////////////////////////////////////////////////////////////////
633
+ # cluster wait for barrier init
634
+ pipeline_init_wait(cluster_shape_mn=self.cluster_shape_mn)
635
+ # /////////////////////////////////////////////////////////////////////////////
636
+ # Prefetch
637
+ # /////////////////////////////////////////////////////////////////////////////
638
+ k_tile_cnt = cute.size(gA_mkl, mode=[2])
639
+ prefetch_k_tile_cnt = cutlass.max(cutlass.min(self.ab_stage, k_tile_cnt), 0)
640
+
641
+ mainloop_producer_state = pipeline.make_pipeline_state(
642
+ pipeline.PipelineUserType.Producer, self.ab_stage
643
+ )
644
+ if warp_idx == 0:
645
+ # /////////////////////////////////////////////////////////////////////////////
646
+ # Prefetch TMA load
647
+ # /////////////////////////////////////////////////////////////////////////////
648
+ for prefetch_idx in cutlass.range(prefetch_k_tile_cnt, unroll=1):
649
+ # /////////////////////////////////////////////////////////////////////////////
650
+ # Wait for A/B buffers to be empty before loading into them
651
+ # Also sets the transaction barrier for the A/B buffers
652
+ # /////////////////////////////////////////////////////////////////////////////
653
+ mainloop_pipeline.producer_acquire(mainloop_producer_state)
654
+ # /////////////////////////////////////////////////////////////////////////////
655
+ # Slice to global/shared memref to current k_tile
656
+ # /////////////////////////////////////////////////////////////////////////////
657
+ tAgA_k = tAgA_mkl[(None, mainloop_producer_state.count)]
658
+ tAsA_pipe = tAsA[(None, mainloop_producer_state.index)]
659
+
660
+ tBgB_k = tBgB_nkl[(None, mainloop_producer_state.count)]
661
+ tBsB_pipe = tBsB[(None, mainloop_producer_state.index)]
662
+
663
+ # /////////////////////////////////////////////////////////////////////////////
664
+ # TMA load A/B
665
+ # /////////////////////////////////////////////////////////////////////////////
666
+ cute.copy(
667
+ tma_atom_a,
668
+ tAgA_k,
669
+ tAsA_pipe,
670
+ tma_bar_ptr=mainloop_pipeline.producer_get_barrier(
671
+ mainloop_producer_state
672
+ ),
673
+ mcast_mask=a_mcast_mask,
674
+ )
675
+ cute.copy(
676
+ tma_atom_b,
677
+ tBgB_k,
678
+ tBsB_pipe,
679
+ tma_bar_ptr=mainloop_pipeline.producer_get_barrier(
680
+ mainloop_producer_state
681
+ ),
682
+ mcast_mask=b_mcast_mask,
683
+ )
684
+ # Mainloop pipeline's producer commit is a NOP
685
+ mainloop_pipeline.producer_commit(mainloop_producer_state)
686
+ mainloop_producer_state.advance()
687
+
688
+ # /////////////////////////////////////////////////////////////////////////////
689
+ # Prologue MMAs
690
+ # /////////////////////////////////////////////////////////////////////////////
691
+ k_pipe_mmas = 1
692
+
693
+ mainloop_consumer_read_state = pipeline.make_pipeline_state(
694
+ pipeline.PipelineUserType.Consumer, self.ab_stage
695
+ )
696
+ mainloop_consumer_release_state = pipeline.make_pipeline_state(
697
+ pipeline.PipelineUserType.Consumer, self.ab_stage
698
+ )
699
+
700
+ peek_ab_full_status = cutlass.Boolean(1)
701
+ if mainloop_consumer_read_state.count < k_tile_cnt:
702
+ peek_ab_full_status = mainloop_pipeline.consumer_try_wait(
703
+ mainloop_consumer_read_state
704
+ )
705
+
706
+ tiled_mma.set(cute.nvgpu.warpgroup.Field.ACCUMULATE, False)
707
+ num_k_blocks = cute.size(tCrA, mode=[2])
708
+ for k_tile in cutlass.range_constexpr(k_pipe_mmas):
709
+ # Wait for A/B buffer to be ready
710
+ mainloop_pipeline.consumer_wait(
711
+ mainloop_consumer_read_state, peek_ab_full_status
712
+ )
713
+
714
+ cute.nvgpu.warpgroup.fence()
715
+ for k_block_idx in cutlass.range(num_k_blocks, unroll_full=True):
716
+ k_block_coord = (
717
+ None,
718
+ None,
719
+ k_block_idx,
720
+ mainloop_consumer_read_state.index,
721
+ )
722
+ tCrA_1phase = tCrA[k_block_coord]
723
+ tCrB_1phase = tCrB[k_block_coord]
724
+
725
+ cute.gemm(
726
+ tiled_mma,
727
+ accumulators,
728
+ tCrA_1phase,
729
+ tCrB_1phase,
730
+ accumulators,
731
+ )
732
+ tiled_mma.set(cute.nvgpu.warpgroup.Field.ACCUMULATE, True)
733
+
734
+ cute.nvgpu.warpgroup.commit_group()
735
+ mainloop_consumer_read_state.advance()
736
+ peek_ab_full_status = cutlass.Boolean(1)
737
+ if mainloop_consumer_read_state.count < k_tile_cnt:
738
+ peek_ab_full_status = mainloop_pipeline.consumer_try_wait(
739
+ mainloop_consumer_read_state
740
+ )
741
+
742
+ # /////////////////////////////////////////////////////////////////////////////
743
+ # MAINLOOP
744
+ # /////////////////////////////////////////////////////////////////////////////
745
+ for k_tile in cutlass.range(k_pipe_mmas, k_tile_cnt, 1, unroll=1):
746
+ # /////////////////////////////////////////////////////////////////////////////
747
+ # Wait for TMA copies to complete
748
+ # /////////////////////////////////////////////////////////////////////////////
749
+ mainloop_pipeline.consumer_wait(
750
+ mainloop_consumer_read_state, peek_ab_full_status
751
+ )
752
+ # /////////////////////////////////////////////////////////////////////////////
753
+ # WGMMA
754
+ # /////////////////////////////////////////////////////////////////////////////
755
+ cute.nvgpu.warpgroup.fence()
756
+ for k_block_idx in cutlass.range(num_k_blocks, unroll_full=True):
757
+ k_block_coord = (
758
+ None,
759
+ None,
760
+ k_block_idx,
761
+ mainloop_consumer_read_state.index,
762
+ )
763
+ tCrA_1phase = tCrA[k_block_coord]
764
+ tCrB_1phase = tCrB[k_block_coord]
765
+
766
+ cute.gemm(
767
+ tiled_mma,
768
+ accumulators,
769
+ tCrA_1phase,
770
+ tCrB_1phase,
771
+ accumulators,
772
+ )
773
+
774
+ cute.nvgpu.warpgroup.commit_group()
775
+ # Wait on the wgmma barrier for previous k_pipe_mmas wgmmas to complete
776
+ cute.nvgpu.warpgroup.wait_group(k_pipe_mmas)
777
+
778
+ mainloop_pipeline.consumer_release(mainloop_consumer_release_state)
779
+
780
+ mainloop_consumer_read_state.advance()
781
+ mainloop_consumer_release_state.advance()
782
+
783
+ peek_ab_full_status = cutlass.Boolean(1)
784
+ if mainloop_consumer_read_state.count < k_tile_cnt:
785
+ peek_ab_full_status = mainloop_pipeline.consumer_try_wait(
786
+ mainloop_consumer_read_state
787
+ )
788
+ # /////////////////////////////////////////////////////////////////////////////
789
+ # TMA load
790
+ # /////////////////////////////////////////////////////////////////////////////
791
+ if warp_idx == 0 and mainloop_producer_state.count < k_tile_cnt:
792
+ # /////////////////////////////////////////////////////////////////////////////
793
+ # Wait for A/B buffers to be empty before loading into them
794
+ # Also sets the transaction barrier for the A/B buffers
795
+ # /////////////////////////////////////////////////////////////////////////////
796
+ mainloop_pipeline.producer_acquire(mainloop_producer_state)
797
+
798
+ # /////////////////////////////////////////////////////////////////////////////
799
+ # Slice to global/shared memref to current k_tile
800
+ # /////////////////////////////////////////////////////////////////////////////
801
+ tAgA_k = tAgA_mkl[(None, mainloop_producer_state.count)]
802
+ tAsA_pipe = tAsA[(None, mainloop_producer_state.index)]
803
+
804
+ tBgB_k = tBgB_nkl[(None, mainloop_producer_state.count)]
805
+ tBsB_pipe = tBsB[(None, mainloop_producer_state.index)]
806
+
807
+ # /////////////////////////////////////////////////////////////////////////////
808
+ # TMA load A/B
809
+ # /////////////////////////////////////////////////////////////////////////////
810
+ cute.copy(
811
+ tma_atom_a,
812
+ tAgA_k,
813
+ tAsA_pipe,
814
+ tma_bar_ptr=mainloop_pipeline.producer_get_barrier(
815
+ mainloop_producer_state
816
+ ),
817
+ mcast_mask=a_mcast_mask,
818
+ )
819
+ cute.copy(
820
+ tma_atom_b,
821
+ tBgB_k,
822
+ tBsB_pipe,
823
+ tma_bar_ptr=mainloop_pipeline.producer_get_barrier(
824
+ mainloop_producer_state
825
+ ),
826
+ mcast_mask=b_mcast_mask,
827
+ )
828
+ # Mainloop pipeline's producer commit is a NOP
829
+ mainloop_pipeline.producer_commit(mainloop_producer_state)
830
+ mainloop_producer_state.advance()
831
+
832
+ # /////////////////////////////////////////////////////////////////////////////
833
+ # EPILOG
834
+ # /////////////////////////////////////////////////////////////////////////////
835
+ cute.nvgpu.warpgroup.wait_group(0)
836
+
837
+ if cute.size(self.cluster_shape_mn) > 1:
838
+ # Wait for all threads in the cluster to finish, avoid early release of smem
839
+ cute.arch.cluster_arrive()
840
+ cute.arch.cluster_wait()
841
+ else:
842
+ # For cluster that has a single thread block, it might have more than one warp groups.
843
+ # Wait for all warp groups in the thread block to finish, because smem for tensor A in
844
+ # the mainloop is reused in the epilogue.
845
+ cute.arch.sync_threads()
846
+
847
+ copy_atom_r2s = sm90_utils.sm90_get_smem_store_op(
848
+ self.c_layout,
849
+ elem_ty_d=self.c_dtype,
850
+ elem_ty_acc=self.acc_dtype,
851
+ )
852
+
853
+ copy_atom_C = cute.make_copy_atom(
854
+ cute.nvgpu.warp.StMatrix8x8x16bOp(
855
+ self.c_layout.is_m_major_c(),
856
+ 4,
857
+ ),
858
+ self.c_dtype,
859
+ )
860
+
861
+ tiled_copy_C_Atom = cute.make_tiled_copy_C_atom(copy_atom_C, tiled_mma)
862
+
863
+ tiled_copy_r2s = cute.make_tiled_copy_S(
864
+ copy_atom_r2s,
865
+ tiled_copy_C_Atom,
866
+ )
867
+
868
+ # (R2S, R2S_M, R2S_N, PIPE_D)
869
+ thr_copy_r2s = tiled_copy_r2s.get_slice(tidx)
870
+ tRS_sD = thr_copy_r2s.partition_D(sC)
871
+ # (R2S, R2S_M, R2S_N)
872
+ tRS_rAcc = tiled_copy_r2s.retile(accumulators)
873
+ tRS_cD = thr_copy_r2s.partition_S(gC_coord)
874
+
875
+ # Allocate D registers.
876
+ rD_shape = cute.shape(thr_copy_r2s.partition_S(sC))
877
+ tRS_rD_layout = cute.make_layout(rD_shape[:3])
878
+ tRS_rD = cute.make_rmem_tensor_like(tRS_rD_layout, self.acc_dtype)
879
+ size_tRS_rD = cute.size(tRS_rD)
880
+
881
+ sepi_for_tma_partition = cute.group_modes(sC, 0, 2)
882
+ tCgC_for_tma_partition = cute.zipped_divide(gC_mnl, self.epi_tile)
883
+
884
+ bSG_sD, bSG_gD = cute.nvgpu.cpasync.tma_partition(
885
+ tma_atom_c,
886
+ 0,
887
+ cute.make_layout(1),
888
+ sepi_for_tma_partition,
889
+ tCgC_for_tma_partition,
890
+ )
891
+
892
+ epi_tile_num = cute.size(tCgC_for_tma_partition, mode=[1])
893
+ epi_tile_shape = tCgC_for_tma_partition.shape[1]
894
+ epi_tile_layout = cute.make_layout(
895
+ epi_tile_shape, stride=(epi_tile_shape[1], 1)
896
+ )
897
+
898
+ # Initialize tma store c_pipeline
899
+ c_producer_group = pipeline.CooperativeGroup(
900
+ pipeline.Agent.Thread, self.threads_per_cta
901
+ )
902
+ c_pipeline = pipeline.PipelineTmaStore.create(
903
+ num_stages=self.epi_stage,
904
+ producer_group=c_producer_group,
905
+ )
906
+
907
+ for epi_idx in cutlass.range_constexpr(epi_tile_num):
908
+ # Copy from accumulators to D registers
909
+ for epi_v in cutlass.range_constexpr(size_tRS_rD):
910
+ tRS_rD[epi_v] = tRS_rAcc[epi_idx * size_tRS_rD + epi_v]
911
+
912
+ tRS_rD_out = cute.make_rmem_tensor_like(tRS_rD_layout, self.c_dtype)
913
+ for epi_v in cutlass.range_constexpr(size_tRS_rD):
914
+ register_idx = epi_idx * size_tRS_rD + epi_v
915
+ global_coord = tRS_cD[register_idx]
916
+ global_m = cute.crd2idx(global_coord[0], cute.make_layout(mC_mnl.shape[0]))
917
+ global_n = cute.crd2idx(global_coord[1], cute.make_layout(mC_mnl.shape[1]))
918
+ if global_m < cute.size(mScaleA) and global_n < cute.size(mScaleB):
919
+ value = tRS_rD[epi_v].to(cutlass.Float32)
920
+ value = mul_rn(mul_rn(value, mScaleA[global_m]), mScaleB[global_n])
921
+ value = value + mBias[global_n].to(cutlass.Float32)
922
+ tRS_rD_out[epi_v] = value.to(self.c_dtype)
923
+ else:
924
+ tRS_rD_out[epi_v] = self.c_dtype(0)
925
+
926
+ # Copy from D registers to shared memory
927
+ epi_buffer = epi_idx % cute.size(tRS_sD, mode=[3])
928
+ cute.copy(
929
+ tiled_copy_r2s, tRS_rD_out, tRS_sD[(None, None, None, epi_buffer)]
930
+ )
931
+
932
+ cute.arch.fence_proxy(
933
+ "async.shared",
934
+ space="cta",
935
+ )
936
+ # barrier for sync
937
+ pipeline.sync(barrier_id=1)
938
+
939
+ gmem_coord = epi_tile_layout.get_hier_coord(epi_idx)
940
+ # Copy from shared memory to global memory
941
+ if warp_idx == 0:
942
+ cute.copy(
943
+ tma_atom_c,
944
+ bSG_sD[(None, epi_buffer)],
945
+ bSG_gD[(None, gmem_coord)],
946
+ )
947
+ c_pipeline.producer_commit()
948
+ c_pipeline.producer_acquire()
949
+
950
+ pipeline.sync(barrier_id=1)
951
+
952
+ if warp_idx == 0:
953
+ c_pipeline.producer_tail()
954
+
955
+ return
956
+
957
+ @staticmethod
958
+ def _compute_stages(
959
+ tile_shape_mnk: tuple[int, int, int],
960
+ a_dtype: type[cutlass.Numeric],
961
+ b_dtype: type[cutlass.Numeric],
962
+ smem_capacity: int,
963
+ occupancy: int,
964
+ ) -> tuple[int, int]:
965
+ """Computes the number of stages for A/B/C operands based on heuristics.
966
+
967
+ :param tile_shape_mnk: The shape (M, N, K) of the CTA tile.
968
+ :type tile_shape_mnk: tuple[int, int, int]
969
+ :param a_dtype: Data type of operand A.
970
+ :type a_dtype: type[cutlass.Numeric]
971
+ :param b_dtype: Data type of operand B.
972
+ :type b_dtype: type[cutlass.Numeric]
973
+ :param smem_capacity: Total available shared memory capacity in bytes.
974
+ :type smem_capacity: int
975
+ :param occupancy: Target number of CTAs per SM (occupancy).
976
+ :type occupancy: int
977
+
978
+ :return: A tuple containing the computed number of stages for:
979
+ (A/B operand stages, epilogue stages)
980
+ :rtype: tuple[int, int]
981
+ """
982
+
983
+ epi_stage = 4
984
+ # epi_smem will reuse smem ab.
985
+ epi_bytes = 0
986
+
987
+ a_shape = cute.slice_(tile_shape_mnk, (None, 0, None))
988
+ b_shape = cute.slice_(tile_shape_mnk, (0, None, None))
989
+ ab_bytes_per_stage = (
990
+ cute.size(a_shape) * a_dtype.width // 8
991
+ + cute.size(b_shape) * b_dtype.width // 8
992
+ )
993
+ mbar_helpers_bytes = 1024
994
+
995
+ ab_stage = (
996
+ smem_capacity // occupancy - mbar_helpers_bytes - epi_bytes
997
+ ) // ab_bytes_per_stage
998
+ return ab_stage, epi_stage
999
+
1000
+ @staticmethod
1001
+ def _make_smem_layouts(
1002
+ tile_shape_mnk: tuple[int, int, int],
1003
+ epi_tile: tuple[int, int],
1004
+ a_dtype: type[cutlass.Numeric],
1005
+ a_layout: utils.LayoutEnum,
1006
+ b_dtype: type[cutlass.Numeric],
1007
+ b_layout: utils.LayoutEnum,
1008
+ ab_stage: int,
1009
+ c_dtype: type[cutlass.Numeric],
1010
+ c_layout: utils.LayoutEnum,
1011
+ epi_stage: int,
1012
+ ) -> tuple[cute.ComposedLayout, cute.ComposedLayout, cute.ComposedLayout]:
1013
+ """Create shared memory layouts for A, B, and C tensors.
1014
+
1015
+ :param tile_shape_mnk: CTA tile shape (M,N,K)
1016
+ :type tile_shape_mnk: Tuple[int, int, int]
1017
+ :param epi_tile: Epilogue tile shape
1018
+ :type epi_tile: Tuple[int, int]
1019
+ :param a_dtype: Data type for matrix A
1020
+ :type a_dtype: type[cutlass.Numeric]
1021
+ :param a_layout: Layout enum for matrix A
1022
+ :type a_layout: utils.LayoutEnum
1023
+ :param b_dtype: Data type for matrix B
1024
+ :type b_dtype: type[cutlass.Numeric]
1025
+ :param b_layout: Layout enum for matrix B
1026
+ :type b_layout: utils.LayoutEnum
1027
+ :param ab_stage: Number of stages for A/B tensors
1028
+ :type ab_stage: int
1029
+ :param c_dtype: Data type for output matrix C
1030
+ :type c_dtype: type[cutlass.Numeric]
1031
+ :param c_layout: Layout enum for the output matrix C
1032
+ :type c_layout: utils.LayoutEnum
1033
+ :param epi_stage: Number of epilogue stages
1034
+ :type epi_stage: int
1035
+
1036
+ :return: Tuple of shared memory layouts for A, B, and C
1037
+ :rtype: Tuple[cute.ComposedLayout, cute.ComposedLayout, cute.ComposedLayout]
1038
+ """
1039
+ a_smem_layout_staged = sm90_utils.make_smem_layout_a(
1040
+ a_layout,
1041
+ tile_shape_mnk,
1042
+ a_dtype,
1043
+ ab_stage,
1044
+ )
1045
+
1046
+ b_smem_layout_staged = sm90_utils.make_smem_layout_b(
1047
+ b_layout,
1048
+ tile_shape_mnk,
1049
+ b_dtype,
1050
+ ab_stage,
1051
+ )
1052
+
1053
+ epi_smem_layout_staged = sm90_utils.make_smem_layout_epi(
1054
+ c_dtype,
1055
+ c_layout,
1056
+ epi_tile,
1057
+ epi_stage,
1058
+ )
1059
+
1060
+ return a_smem_layout_staged, b_smem_layout_staged, epi_smem_layout_staged
1061
+
1062
+ @staticmethod
1063
+ def _compute_grid(
1064
+ c: cute.Tensor,
1065
+ tile_shape_mnk: tuple[int, int, int],
1066
+ cluster_shape_mn: tuple[int, int],
1067
+ ) -> tuple[int, int, int]:
1068
+ """Compute grid shape for the output tensor C.
1069
+
1070
+ :param c: The output tensor C
1071
+ :type c: cute.Tensor
1072
+ :param tile_shape_mnk: The shape (M, N, K) of the CTA tile.
1073
+ :type tile_shape_mnk: tuple[int, int, int]
1074
+ :param cluster_shape_mn: Shape of each cluster in M, N dimensions.
1075
+ :type cluster_shape_mn: tuple[int, int]
1076
+
1077
+ :return: Grid shape for kernel launch.
1078
+ :rtype: tuple[int, int, int]
1079
+ """
1080
+
1081
+ c_shape = (tile_shape_mnk[0], tile_shape_mnk[1])
1082
+ gc = cute.zipped_divide(c, tiler=c_shape)
1083
+ cluster_shape_mnl = (*cluster_shape_mn, 1)
1084
+ clusters = cute.ceil_div(cute.get(gc.layout, mode=[1]).shape, cluster_shape_mnl)
1085
+ grid = tuple(x * y for x, y in zip(clusters, cluster_shape_mnl))
1086
+ return grid
1087
+
1088
+ @staticmethod
1089
+ def _make_tma_store_atoms_and_tensors(
1090
+ tensor_c: cute.Tensor,
1091
+ epi_smem_layout_staged: cute.ComposedLayout,
1092
+ epi_tile: tuple[int, int],
1093
+ ) -> tuple[cute.CopyAtom, cute.Tensor]:
1094
+ """Create TMA atoms and tensors for C tensor storage.
1095
+
1096
+ :param tensor_c: Output tensor C
1097
+ :type tensor_c: cute.Tensor
1098
+ :param epi_smem_layout_staged: Shared memory layout for epilogue
1099
+ :type epi_smem_layout_staged: cute.ComposedLayout
1100
+ :param epi_tile: Epilogue tile shape
1101
+ :type epi_tile: Tuple[int, int]
1102
+
1103
+ :return: TMA atom and tensor for C
1104
+ :rtype: Tuple[cute.CopyAtom, cute.Tensor]
1105
+ """
1106
+ epi_smem_layout = cute.slice_(epi_smem_layout_staged, (None, None, 0))
1107
+ tma_atom_c, tma_tensor_c = cute.nvgpu.cpasync.make_tiled_tma_atom(
1108
+ cute.nvgpu.cpasync.CopyBulkTensorTileS2GOp(),
1109
+ tensor_c,
1110
+ epi_smem_layout,
1111
+ epi_tile,
1112
+ )
1113
+
1114
+ return tma_atom_c, tma_tensor_c
1115
+
1116
+ @staticmethod
1117
+ def _make_tma_atoms_and_tensors(
1118
+ tensor: cute.Tensor,
1119
+ smem_layout_staged: cute.ComposedLayout,
1120
+ smem_tile: tuple[int, int],
1121
+ mcast_dim: int,
1122
+ ) -> tuple[cute.CopyAtom, cute.Tensor]:
1123
+ """Create TMA atoms and tensors for input tensors.
1124
+
1125
+ :param tensor: Input tensor (A or B)
1126
+ :type tensor: cute.Tensor
1127
+ :param smem_layout_staged: Shared memory layout for the tensor
1128
+ :type smem_layout_staged: cute.ComposedLayout
1129
+ :param smem_tile: Shared memory tile shape
1130
+ :type smem_tile: Tuple[int, int]
1131
+ :param mcast_dim: Multicast dimension
1132
+ :type mcast_dim: int
1133
+
1134
+ :return: TMA atom and tensor
1135
+ :rtype: Tuple[cute.CopyAtom, cute.Tensor]
1136
+ """
1137
+ op = (
1138
+ cute.nvgpu.cpasync.CopyBulkTensorTileG2SOp()
1139
+ if mcast_dim == 1
1140
+ else cute.nvgpu.cpasync.CopyBulkTensorTileG2SMulticastOp()
1141
+ )
1142
+
1143
+ smem_layout = cute.slice_(smem_layout_staged, (None, None, 0))
1144
+ tma_atom, tma_tensor = cute.nvgpu.cpasync.make_tiled_tma_atom(
1145
+ op,
1146
+ tensor,
1147
+ smem_layout,
1148
+ smem_tile,
1149
+ num_multicast=mcast_dim,
1150
+ )
1151
+ return tma_atom, tma_tensor
1152
+
1153
+ @staticmethod
1154
+ def is_valid_dtypes(
1155
+ a_dtype: Type[cutlass.Numeric],
1156
+ b_dtype: Type[cutlass.Numeric],
1157
+ acc_dtype: Type[cutlass.Numeric],
1158
+ c_dtype: Type[cutlass.Numeric],
1159
+ a_major: str,
1160
+ b_major: str,
1161
+ ) -> bool:
1162
+ """
1163
+ Check if the dtypes are valid
1164
+
1165
+ :param a_dtype: The data type of tensor A
1166
+ :type a_dtype: Type[cutlass.Numeric]
1167
+ :param b_dtype: The data type of tensor B
1168
+ :type b_dtype: Type[cutlass.Numeric]
1169
+ :param acc_dtype: The data type of the accumulator
1170
+ :type acc_dtype: Type[cutlass.Numeric]
1171
+ :param c_dtype: The data type of the output tensor
1172
+ :type c_dtype: Type[cutlass.Numeric]
1173
+ :param a_major: major mode of tensor A
1174
+ :type a_major: str
1175
+ :param b_major: major mode of tensor B
1176
+ :type b_major: str
1177
+
1178
+ :return: True if the dtypes are valid, False otherwise
1179
+ :rtype: bool
1180
+ """
1181
+ is_valid = True
1182
+
1183
+ valid_ab_dtypes = {
1184
+ cutlass.Float16,
1185
+ cutlass.Float8E4M3FN,
1186
+ cutlass.Float8E5M2,
1187
+ cutlass.Uint8,
1188
+ cutlass.Int8,
1189
+ }
1190
+ if a_dtype not in valid_ab_dtypes:
1191
+ is_valid = False
1192
+ if b_dtype not in valid_ab_dtypes:
1193
+ is_valid = False
1194
+
1195
+ # make sure a_dtype == b_dtype for Float16
1196
+ if a_dtype.width == 16 and a_dtype != b_dtype:
1197
+ is_valid = False
1198
+ if a_dtype.width != b_dtype.width:
1199
+ is_valid = False
1200
+ if not a_dtype.is_same_kind(b_dtype):
1201
+ is_valid = False
1202
+
1203
+ # for 8-bit types, this implementation only supports k-major layout
1204
+ if (a_dtype.width == 8 and a_major != "k") or (
1205
+ b_dtype.width == 8 and b_major != "k"
1206
+ ):
1207
+ is_valid = False
1208
+
1209
+ # Define compatibility mapping between accumulator type and AB type
1210
+ acc_ab_compatibility = {
1211
+ cutlass.Float32: {
1212
+ cutlass.Float16,
1213
+ cutlass.Float8E4M3FN,
1214
+ cutlass.Float8E5M2,
1215
+ },
1216
+ cutlass.Float16: {
1217
+ cutlass.Float16,
1218
+ cutlass.Float8E4M3FN,
1219
+ cutlass.Float8E5M2,
1220
+ },
1221
+ cutlass.Int32: {cutlass.Uint8, cutlass.Int8},
1222
+ }
1223
+ # Check compatibility between accumulator type and A type
1224
+ if a_dtype not in acc_ab_compatibility[acc_dtype]:
1225
+ is_valid = False
1226
+
1227
+ # Define compatibility mapping between accumulator type and C type
1228
+ acc_c_compatibility = {
1229
+ cutlass.Float32: {
1230
+ cutlass.Float32,
1231
+ cutlass.Float16,
1232
+ cutlass.Float8E4M3FN,
1233
+ cutlass.Float8E5M2,
1234
+ },
1235
+ cutlass.Float16: {
1236
+ cutlass.Float32,
1237
+ cutlass.Float16,
1238
+ cutlass.Float8E4M3FN,
1239
+ cutlass.Float8E5M2,
1240
+ },
1241
+ cutlass.Int32: {
1242
+ cutlass.Float32,
1243
+ cutlass.Float16,
1244
+ cutlass.BFloat16,
1245
+ cutlass.Int32,
1246
+ cutlass.Int8,
1247
+ cutlass.Uint8,
1248
+ },
1249
+ }
1250
+ # Check compatibility between accumulator type and C type
1251
+ if c_dtype not in acc_c_compatibility[acc_dtype]:
1252
+ is_valid = False
1253
+
1254
+ return is_valid
1255
+
1256
+ @staticmethod
1257
+ def is_valid_tensor_alignment(
1258
+ m: int,
1259
+ n: int,
1260
+ k: int,
1261
+ l: int, # noqa: E741 - CUTLASS uses L for the batch dimension.
1262
+ ab_dtype: Type[cutlass.Numeric],
1263
+ c_dtype: Type[cutlass.Numeric],
1264
+ a_major: str,
1265
+ b_major: str,
1266
+ c_major: str,
1267
+ ) -> bool:
1268
+ """
1269
+ Check if the tensor alignment is valid
1270
+
1271
+ :param m: The number of rows in the A tensor
1272
+ :type m: int
1273
+ :param n: The number of columns in the B tensor
1274
+ :type n: int
1275
+ :param k: The number of columns in the A tensor
1276
+ :type k: int
1277
+ :param l: The number of columns in the C tensor
1278
+ :type l: int
1279
+ :param ab_dtype: The data type of the A and B operands
1280
+ :type ab_dtype: Type[cutlass.Numeric]
1281
+ :param c_dtype: The data type of the output tensor
1282
+ :type c_dtype: Type[cutlass.Numeric]
1283
+ :param a_major: The major axis of the A tensor
1284
+ :type a_major: str
1285
+ :param b_major: The major axis of the B tensor
1286
+ :type b_major: str
1287
+ :param c_major: The major axis of the C tensor
1288
+ :type c_major: str
1289
+
1290
+ :return: True if the problem shape is valid, False otherwise
1291
+ :rtype: bool
1292
+ """
1293
+ is_valid = True
1294
+
1295
+ def check_contigous_16B_alignment(dtype, is_mode0_major, tensor_shape):
1296
+ major_mode_idx = 0 if is_mode0_major else 1
1297
+ num_major_elements = tensor_shape[major_mode_idx]
1298
+ num_contiguous_elements = 16 * 8 // dtype.width
1299
+ return num_major_elements % num_contiguous_elements == 0
1300
+
1301
+ if (
1302
+ not check_contigous_16B_alignment(ab_dtype, a_major == "m", (m, k, l))
1303
+ or not check_contigous_16B_alignment(ab_dtype, b_major == "n", (n, k, l))
1304
+ or not check_contigous_16B_alignment(c_dtype, c_major == "m", (m, n, l))
1305
+ ):
1306
+ is_valid = False
1307
+ return is_valid
runtime/clef_text_cute.py ADDED
@@ -0,0 +1,341 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Inference-only CuTe pointwise fusions for CLEF's dense text backbone."""
2
+
3
+ import cuda.bindings.driver as cuda
4
+ import cutlass
5
+ import cutlass.cute as cute
6
+ from cutlass.cute.runtime import make_fake_stream, make_fake_tensor
7
+ import torch
8
+
9
+ from clef_vision_norm import mul_rn
10
+
11
+ _CACHE = {}
12
+
13
+
14
+ class TextRMSNorm:
15
+ """Match ATen's FP32 mean reduction: four accumulators, shuffle 1..16."""
16
+
17
+ def __init__(self, width, eps, residual):
18
+ self.width, self.eps, self.residual = width, eps, residual
19
+
20
+ @cute.jit
21
+ def __call__(self, x: cute.Tensor, skip: cute.Tensor, weight: cute.Tensor,
22
+ summed: cute.Tensor, output: cute.Tensor, stream: cuda.CUstream):
23
+ self.kernel(x, skip, weight, summed, output).launch(
24
+ grid=(cute.ceil_div(x.shape[0], 4), 1, 1), block=(128, 1, 1), stream=stream)
25
+
26
+ @cute.kernel
27
+ def kernel(self, x: cute.Tensor, skip: cute.Tensor, weight: cute.Tensor,
28
+ summed: cute.Tensor, output: cute.Tensor):
29
+ tid, _, _ = cute.arch.thread_idx()
30
+ block, _, _ = cute.arch.block_idx()
31
+ lane, row = tid % 32, block * 4 + tid // 32
32
+ if row < x.shape[0]:
33
+ copy = cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), x.element_type, num_bits_per_copy=64)
34
+ accum = cute.make_rmem_tensor((4,), cutlass.Float32)
35
+ for i in cutlass.range_constexpr(4):
36
+ accum[i] = 0.0
37
+ for step in cutlass.range_constexpr(self.width // 128):
38
+ col = cute.assume(lane * 4 + step * 128, divby=4)
39
+ offset = cute.assume(row * self.width + col, divby=4)
40
+ layout = cute.make_layout((4,))
41
+ xr = cute.make_rmem_tensor((4,), x.element_type)
42
+ cute.copy(copy, cute.make_tensor(x.iterator + offset, layout), xr)
43
+ if cutlass.const_expr(self.residual):
44
+ sr = cute.make_rmem_tensor((4,), x.element_type)
45
+ cute.copy(copy, cute.make_tensor(skip.iterator + offset, layout), sr)
46
+ for i in cutlass.range_constexpr(4):
47
+ xr[i] = (xr[i].to(cutlass.Float32) + sr[i].to(cutlass.Float32)).to(x.element_type)
48
+ for i in cutlass.range_constexpr(4):
49
+ value = xr[i].to(cutlass.Float32)
50
+ accum[i] = accum[i] + mul_rn(value, value)
51
+ total = ((accum[0] + accum[1]) + accum[2]) + accum[3]
52
+ for step in cutlass.range_constexpr(5):
53
+ total = total + cute.arch.shuffle_sync_down(total, 1 << step)
54
+ total = cute.arch.shuffle_sync(total, 0)
55
+ mean = mul_rn(total, cutlass.Float32(1.0 / self.width))
56
+ inv = cute.math.rsqrt(mean + self.eps, fastmath=True)
57
+ for step in cutlass.range_constexpr(self.width // 128):
58
+ col = cute.assume(lane * 4 + step * 128, divby=4)
59
+ offset = cute.assume(row * self.width + col, divby=4)
60
+ layout = cute.make_layout((4,))
61
+ xr = cute.make_rmem_tensor((4,), x.element_type)
62
+ wr = cute.make_rmem_tensor((4,), x.element_type)
63
+ cute.copy(copy, cute.make_tensor(x.iterator + offset, layout), xr)
64
+ cute.copy(copy, cute.make_tensor(weight.iterator + col, layout), wr)
65
+ if cutlass.const_expr(self.residual):
66
+ sr = cute.make_rmem_tensor((4,), x.element_type)
67
+ cute.copy(copy, cute.make_tensor(skip.iterator + offset, layout), sr)
68
+ for i in cutlass.range_constexpr(4):
69
+ xr[i] = (xr[i].to(cutlass.Float32) + sr[i].to(cutlass.Float32)).to(x.element_type)
70
+ cute.copy(copy, xr, cute.make_tensor(summed.iterator + offset, layout))
71
+ result = cute.make_rmem_tensor((4,), x.element_type)
72
+ for i in cutlass.range_constexpr(4):
73
+ value = mul_rn(xr[i].to(cutlass.Float32), inv)
74
+ result[i] = mul_rn(value, 1.0 + wr[i].to(cutlass.Float32)).to(x.element_type)
75
+ cute.copy(copy, result, cute.make_tensor(output.iterator + offset, layout))
76
+
77
+
78
+ class TextSwiGLU:
79
+ @cute.jit
80
+ def __call__(self, gate: cute.Tensor, up: cute.Tensor, output: cute.Tensor, stream: cuda.CUstream):
81
+ self.kernel(gate, up, output).launch(
82
+ grid=(cute.ceil_div(cute.size(gate), 2048), 1, 1), block=(256, 1, 1), stream=stream)
83
+
84
+ @cute.kernel
85
+ def kernel(self, gate: cute.Tensor, up: cute.Tensor, output: cute.Tensor):
86
+ tid, _, _ = cute.arch.thread_idx()
87
+ block, _, _ = cute.arch.block_idx()
88
+ offset = cute.assume((block * 256 + tid) * 8, divby=8)
89
+ if offset < cute.size(gate):
90
+ layout = cute.make_layout((8,))
91
+ copy = cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), gate.element_type, num_bits_per_copy=128)
92
+ gr = cute.make_rmem_tensor((8,), gate.element_type)
93
+ ur = cute.make_rmem_tensor((8,), gate.element_type)
94
+ result = cute.make_rmem_tensor((8,), gate.element_type)
95
+ cute.copy(copy, cute.make_tensor(gate.iterator + offset, layout), gr)
96
+ cute.copy(copy, cute.make_tensor(up.iterator + offset, layout), ur)
97
+ for i in cutlass.range_constexpr(8):
98
+ value = gr[i].to(cutlass.Float32)
99
+ activated = value / (1.0 + cute.math.exp(-value))
100
+ activated = activated.to(gate.element_type).to(cutlass.Float32)
101
+ result[i] = mul_rn(activated, ur[i].to(cutlass.Float32)).to(gate.element_type)
102
+ cute.copy(copy, result, cute.make_tensor(output.iterator + offset, layout))
103
+
104
+
105
+ class DeltaGates:
106
+ def __init__(self, heads):
107
+ self.heads = heads
108
+
109
+ @cute.jit
110
+ def __call__(self, a: cute.Tensor, b: cute.Tensor, bias: cute.Tensor, coefficient: cute.Tensor,
111
+ beta: cute.Tensor, g: cute.Tensor, stream: cuda.CUstream):
112
+ self.kernel(a, b, bias, coefficient, beta, g).launch(
113
+ grid=(cute.ceil_div(cute.size(a), 1024), 1, 1), block=(256, 1, 1), stream=stream)
114
+
115
+ @cute.kernel
116
+ def kernel(self, a: cute.Tensor, b: cute.Tensor, bias: cute.Tensor, coefficient: cute.Tensor,
117
+ beta: cute.Tensor, g: cute.Tensor):
118
+ tid, _, _ = cute.arch.thread_idx()
119
+ block, _, _ = cute.arch.block_idx()
120
+ offset = cute.assume((block * 256 + tid) * 4, divby=4)
121
+ head = cute.assume(offset % self.heads, divby=4)
122
+ if offset < cute.size(a):
123
+ layout = cute.make_layout((4,))
124
+ bf16_copy = cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), a.element_type, num_bits_per_copy=64)
125
+ fp32_copy = cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), coefficient.element_type, num_bits_per_copy=128)
126
+ ar, br, dr, result_beta = [cute.make_rmem_tensor((4,), a.element_type) for _ in range(4)]
127
+ cr, result_g = [cute.make_rmem_tensor((4,), coefficient.element_type) for _ in range(2)]
128
+ cute.copy(bf16_copy, cute.make_tensor(a.iterator + offset, layout), ar)
129
+ cute.copy(bf16_copy, cute.make_tensor(b.iterator + offset, layout), br)
130
+ cute.copy(bf16_copy, cute.make_tensor(bias.iterator + head, layout), dr)
131
+ cute.copy(fp32_copy, cute.make_tensor(coefficient.iterator + head, layout), cr)
132
+ for i in cutlass.range_constexpr(4):
133
+ value = ar[i].to(cutlass.Float32) + dr[i].to(cutlass.Float32)
134
+ softplus = value
135
+ if value <= 20.0:
136
+ softplus = cute.math.log1p(cute.math.exp(value))
137
+ result_g[i] = mul_rn(cr[i], softplus)
138
+ result_beta[i] = (1.0 / (1.0 + cute.math.exp(-br[i].to(cutlass.Float32)))).to(a.element_type)
139
+ cute.copy(bf16_copy, result_beta, cute.make_tensor(beta.iterator + offset, layout))
140
+ cute.copy(fp32_copy, result_g, cute.make_tensor(g.iterator + offset, layout))
141
+
142
+
143
+ def _validate(*tensors):
144
+ device = tensors[0].device
145
+ if torch.is_grad_enabled():
146
+ raise RuntimeError("CLEF text CuTe kernels are inference-only")
147
+ if any(not t.is_cuda or t.device != device or t.dtype != torch.bfloat16
148
+ or not t.is_contiguous() or t.data_ptr() % 16 for t in tensors):
149
+ raise ValueError("Expected aligned, colocated, contiguous CUDA BF16 tensors")
150
+
151
+
152
+ def rms_norm(x, norm, skip=None):
153
+ _validate(x, norm.weight, *(() if skip is None else (skip,)))
154
+ width = x.shape[-1]
155
+ rows = x.numel() // width
156
+ if width not in {256, 5120} or rows < 16 or norm.weight.shape != (width,):
157
+ raise ValueError("The exact CLEF RMSNorm path requires width 256/5120 and at least 16 rows")
158
+ if skip is not None and skip.shape != x.shape:
159
+ raise ValueError("Residual shape must match the input")
160
+ key = ("rms", x.device.index, rows, width, norm.eps, skip is not None)
161
+ fn = _CACHE.get(key)
162
+ if fn is None:
163
+ kwargs = {"memspace": cute.AddressSpace.gmem, "assumed_align": 16}
164
+ matrix = make_fake_tensor(cutlass.BFloat16, (rows, width), stride=(width, 1), **kwargs)
165
+ weight = make_fake_tensor(cutlass.BFloat16, (width,), stride=(1,), **kwargs)
166
+ fn = cute.compile(TextRMSNorm(width, norm.eps, skip is not None), matrix, matrix,
167
+ weight, matrix, matrix, make_fake_stream(use_tvm_ffi_env_stream=True),
168
+ options="--enable-tvm-ffi")
169
+ _CACHE[key] = fn
170
+ output = torch.empty_like(x)
171
+ summed = torch.empty_like(x) if skip is not None else x
172
+ fn(x.view(rows, width), (skip if skip is not None else x).view(rows, width), norm.weight,
173
+ summed.view(rows, width), output.view(rows, width))
174
+ return (summed, output) if skip is not None else output
175
+
176
+
177
+ def swiglu(gate, up):
178
+ _validate(gate, up)
179
+ if gate.shape != up.shape or gate.numel() == 0 or gate.numel() % 8:
180
+ raise ValueError("Expected equal nonempty SwiGLU inputs with a multiple of eight elements")
181
+ key = ("swiglu", gate.device.index, gate.numel())
182
+ fn = _CACHE.get(key)
183
+ if fn is None:
184
+ tensor = make_fake_tensor(cutlass.BFloat16, (gate.numel(),), stride=(1,),
185
+ memspace=cute.AddressSpace.gmem, assumed_align=16)
186
+ fn = cute.compile(TextSwiGLU(), tensor, tensor, tensor,
187
+ make_fake_stream(use_tvm_ffi_env_stream=True), options="--enable-tvm-ffi")
188
+ _CACHE[key] = fn
189
+ output = torch.empty_like(gate)
190
+ fn(gate.flatten(), up.flatten(), output.flatten())
191
+ return output
192
+
193
+
194
+ def delta_gates(a, b, bias, coefficient):
195
+ _validate(a, b, bias)
196
+ heads = bias.numel()
197
+ if (a.shape != b.shape or a.shape[-1] != heads or heads % 4
198
+ or coefficient.shape != bias.shape or coefficient.dtype != torch.float32
199
+ or coefficient.device != a.device or not coefficient.is_contiguous()):
200
+ raise ValueError("Expected matching delta gate inputs and FP32 decay coefficients")
201
+ key = ("gates", a.device.index, a.numel(), heads)
202
+ fn = _CACHE.get(key)
203
+ if fn is None:
204
+ kwargs = {"memspace": cute.AddressSpace.gmem, "assumed_align": 16}
205
+ inp = make_fake_tensor(cutlass.BFloat16, (a.numel(),), stride=(1,), **kwargs)
206
+ dt = make_fake_tensor(cutlass.BFloat16, (heads,), stride=(1,), **kwargs)
207
+ coef = make_fake_tensor(cutlass.Float32, (heads,), stride=(1,), **kwargs)
208
+ gout = make_fake_tensor(cutlass.Float32, (a.numel(),), stride=(1,), **kwargs)
209
+ fn = cute.compile(DeltaGates(heads), inp, inp, dt, coef, inp, gout,
210
+ make_fake_stream(use_tvm_ffi_env_stream=True), options="--enable-tvm-ffi")
211
+ _CACHE[key] = fn
212
+ beta, g = torch.empty_like(b), torch.empty_like(a, dtype=torch.float32)
213
+ fn(a.flatten(), b.flatten(), bias, coefficient, beta.flatten(), g.flatten())
214
+ return beta, g
215
+
216
+
217
+ def patch_text(text, *, rms=True, activation=True, residual=True, gva=False, gates=False, heads=False,
218
+ share_mlp_quant=False, share_attn_quant=False):
219
+ """Patch immutable evaluated BF16 text weights; return a restoration callable."""
220
+ if text.training or text.config.hidden_size != 5120:
221
+ raise ValueError("Expected CLEF's evaluated 5120-wide text backbone")
222
+ if text.config.hidden_act != "silu":
223
+ raise ValueError("CLEF text fusion requires SiLU activation")
224
+ if gva:
225
+ for layer in text.layers:
226
+ rule = getattr(getattr(layer, "linear_attn", None), "chunk_gated_delta_rule", None)
227
+ if getattr(rule, "__name__", "") == "torch_chunk_gated_delta_rule":
228
+ raise ValueError("Native grouped-value attention requires FLA")
229
+ originals = []
230
+ if share_mlp_quant or share_attn_quant:
231
+ from clef_text_int8 import shared_linear_pair
232
+ stats = {"rms_calls": 0, "swiglu_calls": 0, "residual_calls": 0, "delta_calls": 0,
233
+ "shared_quant_calls": 0, "shared_attn_quant_calls": 0, "fallbacks": 0}
234
+
235
+ def replace(module, forward):
236
+ originals.append((module, module.forward))
237
+ module.forward = forward
238
+
239
+ def supported(x):
240
+ return (not text.training and not torch.is_grad_enabled() and x.is_cuda
241
+ and x.dtype == torch.bfloat16 and x.numel() // x.shape[-1] >= 16)
242
+
243
+ if rms:
244
+ for norm in text.modules():
245
+ if norm.__class__.__name__ != "Qwen3_5RMSNorm" or norm.weight.numel() not in ({256, 5120} if heads else {5120}):
246
+ continue
247
+ original = norm.forward
248
+ def forward(x, norm=norm, original=original):
249
+ if supported(x):
250
+ stats["rms_calls"] += 1
251
+ return rms_norm(x.contiguous(), norm)
252
+ stats["fallbacks"] += 1
253
+ return original(x)
254
+ replace(norm, forward)
255
+ for layer in text.layers:
256
+ if (gva or gates or share_attn_quant) and layer.layer_type == "linear_attention":
257
+ attn = layer.linear_attn
258
+ original_attn = attn.forward
259
+ with torch.inference_mode():
260
+ coefficient = -attn.A_log.float().exp()
261
+ def forward(hidden_states, cache_params=None, attention_mask=None, attn=attn,
262
+ coefficient=coefficient, original=original_attn, **kwargs):
263
+ if cache_params is not None or attn.training or torch.is_grad_enabled() or hidden_states.dtype != torch.bfloat16:
264
+ stats["fallbacks"] += 1
265
+ return original(hidden_states, cache_params=cache_params, attention_mask=attention_mask, **kwargs)
266
+ from transformers.models.qwen3_5.modeling_qwen3_5 import apply_mask_to_padding_states
267
+ hidden_states = apply_mask_to_padding_states(hidden_states, attention_mask)
268
+ batch, seq, _ = hidden_states.shape
269
+ if share_attn_quant:
270
+ mixed, z, shared = shared_linear_pair(hidden_states, attn.in_proj_qkv, attn.in_proj_z)
271
+ stats["shared_attn_quant_calls"] += int(shared)
272
+ else:
273
+ mixed, z = attn.in_proj_qkv(hidden_states), attn.in_proj_z(hidden_states)
274
+ mixed = mixed.transpose(1, 2)
275
+ z = z.reshape(batch, seq, -1, attn.head_v_dim)
276
+ b, a = attn.in_proj_b(hidden_states), attn.in_proj_a(hidden_states)
277
+ if attn.causal_conv1d_fn is not None:
278
+ mixed = attn.causal_conv1d_fn(x=mixed, weight=attn.conv1d.weight.squeeze(1),
279
+ bias=attn.conv1d.bias, activation=attn.activation,
280
+ seq_idx=kwargs.get("seq_idx"))
281
+ else:
282
+ mixed = torch.nn.functional.silu(attn.conv1d(mixed)[:, :, :seq])
283
+ q, k, v = mixed.transpose(1, 2).split((attn.key_dim, attn.key_dim, attn.value_dim), dim=-1)
284
+ q, k = [t.reshape(batch, seq, -1, attn.head_k_dim) for t in (q, k)]
285
+ v = v.reshape(batch, seq, -1, attn.head_v_dim)
286
+ if gates:
287
+ beta, g = delta_gates(a, b, attn.dt_bias, coefficient)
288
+ else:
289
+ beta = b.sigmoid()
290
+ g = -attn.A_log.float().exp() * torch.nn.functional.softplus(a.float() + attn.dt_bias)
291
+ ratio = attn.num_v_heads // attn.num_k_heads
292
+ if not gva and ratio > 1:
293
+ q, k = [t.repeat_interleave(ratio, dim=2) for t in (q, k)]
294
+ core, _ = attn.chunk_gated_delta_rule(q, k, v, g=g, beta=beta, initial_state=None,
295
+ output_final_state=False, use_qk_l2norm_in_kernel=True,
296
+ cu_seqlens=kwargs.get("cu_seq_lens_q"))
297
+ core = attn.norm(core.reshape(-1, attn.head_v_dim), z.reshape(-1, attn.head_v_dim))
298
+ stats["delta_calls"] += 1
299
+ return attn.out_proj(core.reshape(batch, seq, -1))
300
+ replace(attn, forward)
301
+ if activation:
302
+ mlp = layer.mlp
303
+ original_mlp = mlp.forward
304
+ def forward(x, mlp=mlp, original=original_mlp):
305
+ if not supported(x) or mlp.training:
306
+ stats["fallbacks"] += 1
307
+ return original(x)
308
+ stats["swiglu_calls"] += 1
309
+ gate, up = mlp.gate_proj, mlp.up_proj
310
+ if share_mlp_quant:
311
+ gate_output, up_output, shared = shared_linear_pair(x, gate, up)
312
+ stats["shared_quant_calls"] += int(shared)
313
+ else:
314
+ gate_output, up_output = gate(x), up(x)
315
+ return mlp.down_proj(swiglu(gate_output, up_output))
316
+ replace(mlp, forward)
317
+ if rms and residual:
318
+ original = layer.forward
319
+ def forward(hidden_states, position_embeddings, attention_mask=None, position_ids=None,
320
+ past_key_values=None, layer=layer, original=original, **kwargs):
321
+ if not supported(hidden_states) or past_key_values is not None or kwargs.get("use_cache"):
322
+ stats["fallbacks"] += 1
323
+ return original(hidden_states, position_embeddings, attention_mask, position_ids, past_key_values, **kwargs)
324
+ normalized = layer.input_layernorm(hidden_states)
325
+ if layer.layer_type == "linear_attention":
326
+ branch = layer.linear_attn(hidden_states=normalized, cache_params=None,
327
+ attention_mask=attention_mask, **kwargs)
328
+ else:
329
+ branch, _ = layer.self_attn(hidden_states=normalized, position_embeddings=position_embeddings,
330
+ attention_mask=attention_mask, position_ids=position_ids,
331
+ past_key_values=None, **kwargs)
332
+ summed, normalized = rms_norm(hidden_states.contiguous(), layer.post_attention_layernorm, branch.contiguous())
333
+ stats["residual_calls"] += 1
334
+ return summed + layer.mlp(normalized)
335
+ replace(layer, forward)
336
+
337
+ def restore():
338
+ for module, original in reversed(originals):
339
+ module.forward = original
340
+ restore.statistics = stats
341
+ return restore
runtime/clef_text_int8.py ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Experimental CLEF W8A8 ConvRot with the H3 SM90 scaled-GEMM runtime."""
2
+
3
+ import torch
4
+
5
+ _CACHE = {}
6
+
7
+
8
+ def scaled_int8_mm(x, weight, input_scale, weight_scale, bias, *, tile=(128, 256), cluster=(1, 1)):
9
+ import cutlass
10
+ import cutlass.cute as cute
11
+ from cutlass.cute.runtime import make_fake_stream, make_fake_tensor
12
+ from clef_int8_cutlass import HopperWgmmaGemmKernel
13
+
14
+ if torch.is_grad_enabled() or not x.is_cuda or torch.cuda.get_device_capability(x.device) != (9, 0):
15
+ raise ValueError("CLEF ConvRot GEMM requires inference mode and SM90")
16
+ if x.ndim != 2 or weight.ndim != 2:
17
+ raise ValueError("Expected two-dimensional INT8 matrices")
18
+ m, k = x.shape
19
+ n = weight.shape[0]
20
+ if (x.dtype != torch.int8 or weight.dtype != torch.int8 or weight.shape[1] != k
21
+ or not x.is_contiguous() or not weight.is_contiguous() or not m or not n or not k
22
+ or k % 16 or n % 8):
23
+ raise ValueError("Expected aligned INT8 row-major matrices")
24
+ tensors = (weight, input_scale, weight_scale, bias)
25
+ if any(t.device != x.device or not t.is_contiguous() or t.data_ptr() % 16 for t in tensors) or x.data_ptr() % 16:
26
+ raise ValueError("INT8 GEMM tensors must be colocated, contiguous, and 16-byte aligned")
27
+ if (input_scale.dtype != torch.float32 or input_scale.numel() != m
28
+ or weight_scale.dtype != torch.float32 or weight_scale.numel() != n
29
+ or bias.dtype != torch.bfloat16 or bias.shape != (n,)):
30
+ raise ValueError("Expected FP32 row/column scales and a BF16 output bias")
31
+ tile, cluster = tuple(tile), tuple(cluster)
32
+ if tile not in {(64, 128), (128, 128), (128, 256)} or cluster not in {(1, 1), (2, 1), (1, 2)}:
33
+ raise ValueError("Unsupported CLEF INT8 tile/cluster configuration")
34
+ key = (x.device.index, n, k, tile, cluster)
35
+ fn = _CACHE.get(key)
36
+ if fn is None:
37
+ dynamic_m = cute.sym_int()
38
+ kwargs = {"memspace": cute.AddressSpace.gmem, "assumed_align": 16}
39
+ a = make_fake_tensor(cutlass.Int8, (dynamic_m, k, 1), stride=(k, 1, 1), **kwargs)
40
+ b = make_fake_tensor(cutlass.Int8, (n, k, 1), stride=(k, 1, 1), **kwargs)
41
+ c = make_fake_tensor(cutlass.BFloat16, (dynamic_m, n, 1), stride=(n, 1, 1), **kwargs)
42
+ sa = make_fake_tensor(cutlass.Float32, (dynamic_m,), stride=(1,), **kwargs)
43
+ sb = make_fake_tensor(cutlass.Float32, (n,), stride=(1,), **kwargs)
44
+ bias_tensor = make_fake_tensor(cutlass.BFloat16, (n,), stride=(1,), **kwargs)
45
+ fn = cute.compile(HopperWgmmaGemmKernel(cutlass.Int32, tile, cluster),
46
+ a, b, c, sa, sb, bias_tensor, make_fake_stream(use_tvm_ffi_env_stream=True),
47
+ options="--enable-tvm-ffi")
48
+ _CACHE[key] = fn
49
+ output = torch.empty((m, n), device=x.device, dtype=torch.bfloat16)
50
+ fn(x.view(m, k, 1), weight.view(n, k, 1), output.view(m, n, 1),
51
+ input_scale.flatten(), weight_scale.flatten(), bias)
52
+ return output
53
+
54
+
55
+ class ConvRotLinear(torch.nn.Module):
56
+ """Immutable inference projection; rotation/quantization happens once per call."""
57
+
58
+ def __init__(self, linear, *, group_size=256, backend="cute", tile=(128, 256), cluster=(1, 1)):
59
+ super().__init__()
60
+ from comfy_kitchen import _cuda_backend
61
+
62
+ if (group_size != 256 or backend not in {"cute", "torch"} or linear.training
63
+ or torch.is_grad_enabled() or not linear.weight.is_cuda
64
+ or linear.weight.dtype != torch.bfloat16 or linear.in_features % group_size
65
+ or (linear.bias is not None and linear.bias.dtype != torch.bfloat16)):
66
+ raise ValueError("Expected BF16 weights divisible by the ConvRot group")
67
+ self.in_features, self.out_features = linear.in_features, linear.out_features
68
+ self.group_size, self.backend, self.tile, self.cluster = group_size, backend, tile, cluster
69
+ weight, scale = _cuda_backend.quantize_int8_rowwise_convrot64(linear.weight.detach().contiguous(), group_size)
70
+ self.register_buffer("weight", weight)
71
+ self.register_buffer("weight_scale", scale.flatten())
72
+ bias = linear.bias.detach().clone() if linear.bias is not None else torch.zeros(self.out_features, device=weight.device, dtype=torch.bfloat16)
73
+ self.register_buffer("bias", bias)
74
+ self.eval()
75
+
76
+ def quantize_input(self, x):
77
+ from comfy_kitchen import _cuda_backend
78
+
79
+ if (self.training or torch.is_grad_enabled() or x.dtype != torch.bfloat16
80
+ or x.device != self.weight.device or x.ndim < 1 or x.shape[-1] != self.in_features
81
+ or not x.numel()):
82
+ raise ValueError("CLEF ConvRot linears are BF16-input inference-only")
83
+ return _cuda_backend.quantize_int8_rowwise_convrot64(x.reshape(-1, self.in_features).contiguous(), self.group_size)
84
+
85
+ def forward_quantized(self, q, scale):
86
+ if self.training or torch.is_grad_enabled():
87
+ raise ValueError("CLEF ConvRot linears are inference-only")
88
+ if self.backend == "cute":
89
+ output = scaled_int8_mm(q, self.weight, scale, self.weight_scale, self.bias,
90
+ tile=self.tile, cluster=self.cluster)
91
+ elif self.backend == "torch":
92
+ padding = max(24, (q.shape[0] + 7) // 8 * 8) - q.shape[0]
93
+ qp = torch.nn.functional.pad(q, (0, 0, 0, padding)) if padding else q
94
+ acc = torch._int_mm(qp, self.weight.t())[:q.shape[0]]
95
+ output = (acc.float() * scale * self.weight_scale + self.bias.float()).to(torch.bfloat16)
96
+ else:
97
+ raise ValueError("Unknown CLEF ConvRot backend")
98
+ return output
99
+
100
+ def forward(self, x):
101
+ q, scale = self.quantize_input(x)
102
+ return self.forward_quantized(q, scale).view(*x.shape[:-1], self.out_features)
103
+
104
+
105
+ def shared_linear_pair(x, first, second):
106
+ """Reuse a rotation only when both projections have the same quantization contract."""
107
+ if (isinstance(first, ConvRotLinear) and isinstance(second, ConvRotLinear)
108
+ and first.in_features == second.in_features and first.group_size == second.group_size
109
+ and first.weight.device == second.weight.device):
110
+ q, scale = first.quantize_input(x)
111
+ leading = x.shape[:-1]
112
+ return (first.forward_quantized(q, scale).view(*leading, first.out_features),
113
+ second.forward_quantized(q, scale).view(*leading, second.out_features), True)
114
+ return first(x), second(x), False
115
+
116
+
117
+ def quantize_text(text, *, scope="mlp", group_size=256, backend="cute", tile=(128, 256), cluster=(1, 1),
118
+ calibration_index=None):
119
+ """Replace text projections without retaining a second full set of weights."""
120
+ if (text.training or torch.is_grad_enabled() or scope not in {"mlp-gate-up", "mlp", "all"}
121
+ or group_size != 256 or backend not in {"cute", "torch"}):
122
+ raise ValueError("Quantize evaluated text weights in inference mode with scope mlp-gate-up/mlp/all")
123
+ names = []
124
+ checkpoint = None
125
+ if calibration_index is not None:
126
+ from clef_int8_calibration import CalibrationIndex
127
+ checkpoint = CalibrationIndex(calibration_index, text, scope)
128
+ for name, module in text.named_modules():
129
+ if not isinstance(module, torch.nn.Linear) or min(module.in_features, module.out_features) < 1024:
130
+ continue
131
+ if scope != "all" and ".mlp." not in name:
132
+ continue
133
+ if scope == "mlp-gate-up" and name.rsplit(".", 1)[-1] not in {"gate_proj", "up_proj"}:
134
+ continue
135
+ names.append(name)
136
+ original_bytes = quantized_bytes = 0
137
+ for name in names:
138
+ parent_name, leaf = name.rsplit(".", 1)
139
+ parent = text.get_submodule(parent_name)
140
+ linear = getattr(parent, leaf)
141
+ original_bytes += linear.weight.numel() * linear.weight.element_size()
142
+ quantized = (checkpoint.load(name, linear, backend=backend, tile=tile, cluster=cluster)
143
+ if checkpoint is not None else
144
+ ConvRotLinear(linear, group_size=group_size, backend=backend, tile=tile, cluster=cluster))
145
+ quantized_bytes += sum(t.numel() * t.element_size() for t in quantized.buffers())
146
+ setattr(parent, leaf, quantized)
147
+ del linear
148
+ torch.cuda.synchronize()
149
+ torch.cuda.empty_cache()
150
+ return {"scope": scope, "linears": len(names), "group_size": group_size,
151
+ "original_weight_bytes": original_bytes, "quantized_weight_bytes": quantized_bytes,
152
+ "backend": backend, "tile": tile, "cluster": cluster,
153
+ "calibration_index": str(calibration_index) if calibration_index is not None else None,
154
+ "total_quantized_linears": sum(isinstance(module, ConvRotLinear) for module in text.modules())}
runtime/clef_text_metrics.py ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """CPU-only decision drift reporting for CLEF precision experiments."""
2
+
3
+ import math
4
+ import statistics
5
+
6
+
7
+ def decision_drift(actual, expected):
8
+ """Compare nested record/question/choice logits without hiding close flips."""
9
+ squared_error = squared_reference = 0.0
10
+ max_abs = 0.0
11
+ decisions = []
12
+ for record, (candidate, reference) in enumerate(zip(actual, expected, strict=True)):
13
+ for question, (a, b) in enumerate(zip(candidate, reference, strict=True)):
14
+ if len(a) != len(b) or len(b) < 2:
15
+ raise ValueError("Each decision needs matching logits for at least two choices")
16
+ if not all(math.isfinite(value) for value in [*a, *b]):
17
+ raise ValueError("Non-finite CLEF logits")
18
+ order = sorted(range(len(b)), key=lambda i: b[i], reverse=True)
19
+ chosen = max(range(len(a)), key=lambda i: a[i])
20
+ errors = [x - y for x, y in zip(a, b, strict=True)]
21
+ squared_error += sum(value * value for value in errors)
22
+ squared_reference += sum(value * value for value in b)
23
+ error = max(abs(value) for value in errors)
24
+ max_abs = max(max_abs, error)
25
+ margin = b[order[0]] - b[order[1]]
26
+ decisions.append({"record": record, "question": question,
27
+ "reference_choice": order[0], "candidate_choice": chosen,
28
+ "matched": chosen == order[0], "reference_margin": margin,
29
+ "logits_max_abs": error,
30
+ "margin_exceeds_twice_error": margin > 2 * error})
31
+ if not decisions:
32
+ raise ValueError("No CLEF decisions to compare")
33
+ margins = [row["reference_margin"] for row in decisions]
34
+ matched = sum(row["matched"] for row in decisions)
35
+ return {"logits_max_abs": max_abs,
36
+ "logits_relative_l2": math.sqrt(squared_error) / max(math.sqrt(squared_reference), 1e-12),
37
+ "decisions_matched": matched, "decisions_total": len(decisions),
38
+ "decision_agreement": matched / len(decisions),
39
+ "reference_margin_min": min(margins), "reference_margin_median": statistics.median(margins),
40
+ "flips": [row for row in decisions if not row["matched"]], "decisions": decisions}
runtime/clef_text_profile.py ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Opt-in CLEF text traces; instrumentation never participates in warm timings."""
2
+
3
+ from collections import defaultdict
4
+ from contextlib import contextmanager
5
+ import statistics
6
+
7
+ import torch
8
+
9
+
10
+ def text_stages(text, head=None):
11
+ yield "text", text
12
+ for name, module in text.named_modules():
13
+ parts = name.split(".")
14
+ if (len(parts) == 2 and parts[0] == "layers") or (
15
+ name and parts[-1] in {"input_layernorm", "post_attention_layernorm", "norm", "q_norm", "k_norm",
16
+ "mlp", "linear_attn", "self_attn", "gate_proj", "up_proj", "down_proj",
17
+ "q_proj", "k_proj", "v_proj", "o_proj", "in_proj_qkv", "in_proj_z",
18
+ "in_proj_a", "in_proj_b", "out_proj", "conv1d"}):
19
+ yield name, module
20
+ if head is not None:
21
+ yield "decision_head", head
22
+
23
+
24
+ @contextmanager
25
+ def stage_hooks(text, head=None, *, events=False):
26
+ handles, pending, pairs = [], defaultdict(list), defaultdict(list)
27
+ try:
28
+ for name, module in text_stages(text, head):
29
+ def pre(module, args, name=name):
30
+ scope = torch.profiler.record_function("clef." + name)
31
+ scope.__enter__()
32
+ torch.cuda.nvtx.range_push("clef." + name)
33
+ start = torch.cuda.Event(enable_timing=True) if events else None
34
+ if start is not None:
35
+ start.record()
36
+ pending[name].append((scope, start))
37
+
38
+ def post(module, args, output, name=name):
39
+ scope, start = pending[name].pop()
40
+ if start is not None:
41
+ end = torch.cuda.Event(enable_timing=True)
42
+ end.record()
43
+ pairs[name].append((start, end))
44
+ torch.cuda.nvtx.range_pop()
45
+ scope.__exit__(None, None, None)
46
+
47
+ handles.append(module.register_forward_pre_hook(pre))
48
+ handles.append(module.register_forward_hook(post, always_call=True))
49
+ yield pairs
50
+ finally:
51
+ for handle in handles:
52
+ handle.remove()
53
+
54
+
55
+ def summarize_kernels(prof, iterations):
56
+ kernels, transfers = defaultdict(list), defaultdict(list)
57
+ for event in prof.events():
58
+ if event.device_type != torch.autograd.DeviceType.CUDA:
59
+ continue
60
+ target = transfers if event.name.lower().startswith(("memcpy", "memset")) else kernels
61
+ target[event.name].append(event.time_range.elapsed_us() / 1000)
62
+ rows = lambda source: sorted(
63
+ [{"name": name, "total_per_forward_ms": sum(values) / iterations,
64
+ "median_call_ms": statistics.median(values), "calls_per_forward": len(values) / iterations}
65
+ for name, values in source.items()], key=lambda row: -row["total_per_forward_ms"])
66
+ return {"kernels": rows(kernels), "transfers": rows(transfers),
67
+ "kernel_sum_ms": sum(map(sum, kernels.values())) / iterations,
68
+ "kernel_count_per_forward": sum(map(len, kernels.values())) / iterations}
69
+
70
+
71
+ def profile_text(fn, text, directory, *, head=None, iterations=3, capture_nsys=False):
72
+ if iterations < 1:
73
+ raise ValueError("Profile iterations must be positive")
74
+ directory.mkdir(parents=True, exist_ok=True)
75
+ torch.cuda.synchronize()
76
+ if capture_nsys:
77
+ torch.cuda.cudart().cudaProfilerStart()
78
+ try:
79
+ with stage_hooks(text, head):
80
+ for _ in range(iterations):
81
+ fn()
82
+ torch.cuda.synchronize()
83
+ finally:
84
+ torch.cuda.cudart().cudaProfilerStop()
85
+ return {"capture": "nsys", "instrumented": True}
86
+ activities = [torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA]
87
+ kernel_trace = directory / "kernels.trace.json"
88
+ with torch.profiler.profile(activities=activities) as kernels:
89
+ for _ in range(iterations):
90
+ fn()
91
+ kernels.export_chrome_trace(str(kernel_trace))
92
+ stage_trace = directory / "stages.trace.json"
93
+ with stage_hooks(text, head, events=True) as pairs:
94
+ with torch.profiler.profile(activities=activities, record_shapes=True) as stages:
95
+ for _ in range(iterations):
96
+ fn()
97
+ torch.cuda.synchronize()
98
+ stages.export_chrome_trace(str(stage_trace))
99
+ spans = []
100
+ for name, events in pairs.items():
101
+ values = [start.elapsed_time(end) for start, end in events]
102
+ spans.append({"name": name, "total_per_forward_ms": sum(values) / iterations,
103
+ "median_call_ms": statistics.median(values), "calls_per_forward": len(values) / iterations})
104
+ return {**summarize_kernels(kernels, iterations), "kernel_trace": str(kernel_trace),
105
+ "stage_trace": str(stage_trace), "stages": sorted(spans, key=lambda row: -row["total_per_forward_ms"]),
106
+ "instrumented": True, "stage_spans_overlap": True, "stage_spans_include_launch_gaps": True,
107
+ "functional_fusions_are_attributed_to_parent_modules": True}
runtime/clef_vision_norm.py ADDED
@@ -0,0 +1,217 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """CuTeDSL residual + LayerNorm for CLEF's 1152-wide vision blocks."""
2
+
3
+ import operator
4
+
5
+ import cuda.bindings.driver as cuda
6
+ import cutlass
7
+ import cutlass.cute as cute
8
+ import torch
9
+ from cutlass.cute.runtime import make_fake_stream, make_fake_tensor
10
+ from cutlass.cutlass_dsl import dsl_user_op
11
+ from cutlass._mlir.dialects import llvm
12
+ from cutlass._mlir.extras import types as T
13
+
14
+ _CACHE = {}
15
+
16
+
17
+ @dsl_user_op
18
+ def mul_rn(a, b, *, loc=None, ip=None):
19
+ return cutlass.Float32(llvm.inline_asm(T.f32(), [v.ir_value(loc=loc, ip=ip) for v in (a, b)],
20
+ "mul.rn.f32 $0, $1, $2;", "=f,f,f", has_side_effects=False, is_align_stack=False,
21
+ asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip))
22
+
23
+
24
+ @dsl_user_op
25
+ def fma_rn(a, b, c, *, loc=None, ip=None):
26
+ return cutlass.Float32(llvm.inline_asm(T.f32(), [v.ir_value(loc=loc, ip=ip) for v in (a, b, c)],
27
+ "fma.rn.f32 $0, $1, $2, $3;", "=f,f,f,f", has_side_effects=False, is_align_stack=False,
28
+ asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip))
29
+
30
+
31
+ @dsl_user_op
32
+ def reciprocal_rn(a, *, loc=None, ip=None):
33
+ return cutlass.Float32(llvm.inline_asm(T.f32(), [a.ir_value(loc=loc, ip=ip)],
34
+ "rcp.rn.f32 $0, $1;", "=f,f", has_side_effects=False, is_align_stack=False,
35
+ asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip))
36
+
37
+
38
+ @cute.jit
39
+ def combine_welford(mean_b, var_b, count_b, mean_a, var_a, count_a):
40
+ delta = mean_b - mean_a
41
+ count = count_a + count_b
42
+ mean, var = cutlass.Float32(0), cutlass.Float32(0)
43
+ if count > 0:
44
+ coef = reciprocal_rn(count)
45
+ na = mul_rn(count_a, coef)
46
+ nb = mul_rn(count_b, coef)
47
+ mean = fma_rn(na, mean_a, mul_rn(nb, mean_b))
48
+ product = mul_rn(mul_rn(delta, delta), count_a)
49
+ var = fma_rn(product, nb, var_a + var_b)
50
+ return mean, var, count
51
+
52
+
53
+ class WelfordResidualLayerNorm:
54
+ """Match ATen's vec4 / 128-thread Welford tree before BF16 conversion."""
55
+
56
+ def __init__(self, width, eps):
57
+ self.width, self.eps = width, eps
58
+ self.iterations = (width + 511) // 512
59
+
60
+ @cute.jit
61
+ def __call__(self, x: cute.Tensor, skip: cute.Tensor, weight: cute.Tensor,
62
+ bias: cute.Tensor, residual: cute.Tensor, output: cute.Tensor,
63
+ stream: cuda.CUstream):
64
+ self.kernel(x, skip, weight, bias, residual, output).launch(
65
+ grid=(x.shape[0], 1, 1), block=(128, 1, 1), stream=stream)
66
+
67
+ @cute.kernel
68
+ def kernel(self, x: cute.Tensor, skip: cute.Tensor, weight: cute.Tensor,
69
+ bias: cute.Tensor, residual: cute.Tensor, output: cute.Tensor):
70
+ tid, _, _ = cute.arch.thread_idx()
71
+ row, _, _ = cute.arch.block_idx()
72
+ lane, warp = tid % 32, tid // 32
73
+ smem = cutlass.utils.SmemAllocator()
74
+ partial = smem.allocate_tensor(cutlass.Float32, cute.make_layout((4, 3)), byte_alignment=16)
75
+ values = cute.make_rmem_tensor((self.iterations * 4,), cutlass.Float32)
76
+ copy = cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), x.element_type, num_bits_per_copy=64)
77
+ mean, var, count = cutlass.Float32(0), cutlass.Float32(0), cutlass.Float32(0)
78
+ for step in cutlass.range_constexpr(self.iterations):
79
+ if tid * 4 + step * 512 < self.width:
80
+ base = cute.assume(tid * 4 + step * 512, divby=4)
81
+ offset = cute.assume(row * self.width + base, divby=4)
82
+ layout = cute.make_layout((4,))
83
+ xv = cute.make_tensor(x.iterator + offset, layout)
84
+ sv = cute.make_tensor(skip.iterator + offset, layout)
85
+ rv = cute.make_tensor(residual.iterator + offset, layout)
86
+ xr = cute.make_rmem_tensor((4,), x.element_type)
87
+ sr = cute.make_rmem_tensor((4,), x.element_type)
88
+ rr = cute.make_rmem_tensor((4,), x.element_type)
89
+ cute.copy(copy, xv, xr)
90
+ cute.copy(copy, sv, sr)
91
+ for i in cutlass.range_constexpr(4):
92
+ value = xr[i].to(cutlass.Float32) + sr[i].to(cutlass.Float32)
93
+ value = value.to(x.element_type).to(cutlass.Float32)
94
+ values[step * 4 + i] = value
95
+ rr[i] = value.to(x.element_type)
96
+ delta = value - mean
97
+ count = count + 1.0
98
+ mean = fma_rn(delta, reciprocal_rn(count), mean)
99
+ var = fma_rn(delta, value - mean, var)
100
+ cute.copy(copy, rr, rv)
101
+ for offset in cutlass.range_constexpr(5):
102
+ distance = 16 >> offset
103
+ other_mean = cute.arch.shuffle_sync_down(mean, distance)
104
+ other_var = cute.arch.shuffle_sync_down(var, distance)
105
+ other_count = cute.arch.shuffle_sync_down(count, distance)
106
+ mean, var, count = combine_welford(mean, var, count, other_mean, other_var, other_count)
107
+ if lane == 0:
108
+ partial[warp, 0], partial[warp, 1], partial[warp, 2] = mean, var, count
109
+ cute.arch.sync_threads()
110
+ if tid == 0:
111
+ m0, v0, c0 = combine_welford(partial[0, 0], partial[0, 1], partial[0, 2],
112
+ partial[2, 0], partial[2, 1], partial[2, 2])
113
+ m1, v1, c1 = combine_welford(partial[1, 0], partial[1, 1], partial[1, 2],
114
+ partial[3, 0], partial[3, 1], partial[3, 2])
115
+ mean, var, count = combine_welford(m0, v0, c0, m1, v1, c1)
116
+ partial[0, 0] = mean
117
+ partial[0, 1] = var / self.width
118
+ cute.arch.sync_threads()
119
+ mean = partial[0, 0]
120
+ inv = cute.math.rsqrt(partial[0, 1] + self.eps, fastmath=True)
121
+ for step in cutlass.range_constexpr(self.iterations):
122
+ if tid * 4 + step * 512 < self.width:
123
+ base = cute.assume(tid * 4 + step * 512, divby=4)
124
+ offset = cute.assume(row * self.width + base, divby=4)
125
+ layout = cute.make_layout((4,))
126
+ wv = cute.make_tensor(weight.iterator + base, layout)
127
+ bv = cute.make_tensor(bias.iterator + base, layout)
128
+ ov = cute.make_tensor(output.iterator + offset, layout)
129
+ wr = cute.make_rmem_tensor((4,), x.element_type)
130
+ br = cute.make_rmem_tensor((4,), x.element_type)
131
+ result = cute.make_rmem_tensor((4,), x.element_type)
132
+ cute.copy(copy, wv, wr)
133
+ cute.copy(copy, bv, br)
134
+ for i in cutlass.range_constexpr(4):
135
+ value = mul_rn(inv, values[step * 4 + i] - mean)
136
+ value = fma_rn(wr[i].to(cutlass.Float32), value, br[i].to(cutlass.Float32))
137
+ result[i] = value.to(x.element_type)
138
+ cute.copy(copy, result, ov)
139
+
140
+
141
+ class ResidualLayerNorm:
142
+ def __init__(self, width, eps):
143
+ self.width = width
144
+ self.eps = eps
145
+
146
+ @cute.jit
147
+ def __call__(self, x: cute.Tensor, skip: cute.Tensor, weight: cute.Tensor,
148
+ bias: cute.Tensor, residual: cute.Tensor, output: cute.Tensor,
149
+ stream: cuda.CUstream):
150
+ self.kernel(x, skip, weight, bias, residual, output).launch(
151
+ grid=(cute.ceil_div(x.shape[0], 4), 1, 1), block=(128, 1, 1), stream=stream)
152
+
153
+ @cute.kernel
154
+ def kernel(self, x: cute.Tensor, skip: cute.Tensor, weight: cute.Tensor,
155
+ bias: cute.Tensor, residual: cute.Tensor, output: cute.Tensor):
156
+ tid, _, _ = cute.arch.thread_idx()
157
+ block, _, _ = cute.arch.block_idx()
158
+ row = block * 4 + tid // 32
159
+ lane = tid % 32
160
+ if row < x.shape[0]:
161
+ values = cute.make_rmem_tensor((self.width // 32,), cutlass.Float32)
162
+ total = cutlass.Float32(0)
163
+ for i in cutlass.range_constexpr(self.width // 32):
164
+ col = lane + i * 32
165
+ value = (x[row, col].to(cutlass.Float32) + skip[row, col].to(cutlass.Float32))
166
+ value = value.to(x.element_type).to(cutlass.Float32)
167
+ values[i] = value
168
+ total = total + value
169
+ residual[row, col] = value.to(x.element_type)
170
+ mean = cute.arch.warp_reduction(total, operator.add) / self.width
171
+ variance = cutlass.Float32(0)
172
+ for i in cutlass.range_constexpr(self.width // 32):
173
+ centered = values[i] - mean
174
+ variance = variance + centered * centered
175
+ variance = cute.arch.warp_reduction(variance, operator.add) / self.width
176
+ inv = cute.math.rsqrt(variance + self.eps, fastmath=True)
177
+ for i in cutlass.range_constexpr(self.width // 32):
178
+ col = lane + i * 32
179
+ value = (values[i] - mean) * inv
180
+ value = value * weight[col].to(cutlass.Float32) + bias[col].to(cutlass.Float32)
181
+ output[row, col] = value.to(x.element_type)
182
+
183
+
184
+ def residual_layer_norm(x, skip, norm, *, algorithm="welford"):
185
+ if torch.is_grad_enabled():
186
+ raise RuntimeError("CLEF CuTe normalization is inference-only")
187
+ if algorithm not in {"welford", "two-pass"}:
188
+ raise ValueError("Unknown residual LayerNorm algorithm")
189
+ if x.ndim != 2 or x.shape != skip.shape or x.shape[1] % 32:
190
+ raise ValueError("Expected equal matrices with width divisible by 32")
191
+ if x.dtype != torch.bfloat16 or skip.dtype != x.dtype or not x.is_contiguous() or not skip.is_contiguous():
192
+ raise ValueError("Expected contiguous BF16 residual inputs")
193
+ if not x.is_cuda or skip.device != x.device:
194
+ raise ValueError("Residual inputs must be on the same CUDA device")
195
+ m, n = x.shape
196
+ if m == 0 or norm.weight is None or norm.bias is None:
197
+ raise ValueError("Expected nonempty inputs and affine LayerNorm")
198
+ if tuple(norm.normalized_shape) != (n,):
199
+ raise ValueError("LayerNorm width must match the residual inputs")
200
+ for parameter in (norm.weight, norm.bias):
201
+ if (parameter.shape != (n,) or parameter.device != x.device
202
+ or parameter.dtype != x.dtype or not parameter.is_contiguous()):
203
+ raise ValueError("Expected colocated contiguous BF16 LayerNorm parameters")
204
+ key = (x.device.index, m, n, norm.eps, algorithm)
205
+ fn = _CACHE.get(key)
206
+ if fn is None:
207
+ kwargs = {"memspace": cute.AddressSpace.gmem, "assumed_align": 16}
208
+ matrix = make_fake_tensor(cutlass.BFloat16, (m, n), stride=(n, 1), **kwargs)
209
+ vector = make_fake_tensor(cutlass.BFloat16, (n,), stride=(1,), **kwargs)
210
+ kernel = WelfordResidualLayerNorm if algorithm == "welford" else ResidualLayerNorm
211
+ fn = cute.compile(kernel(n, norm.eps), matrix, matrix, vector, vector,
212
+ matrix, matrix, make_fake_stream(use_tvm_ffi_env_stream=True),
213
+ options="--enable-tvm-ffi")
214
+ _CACHE[key] = fn
215
+ residual, output = torch.empty_like(x), torch.empty_like(x)
216
+ fn(x, skip, norm.weight, norm.bias, residual, output)
217
+ return residual, output
runtime/test_clef_int8_calibration.py ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import tempfile
3
+ import unittest
4
+
5
+ try:
6
+ import torch
7
+ from clef_int8_calibration import CalibrationIndex, CalibrationWriter, fit_affine
8
+ except ModuleNotFoundError as exc:
9
+ if exc.name not in {"torch", "safetensors"}:
10
+ raise
11
+ torch = None
12
+
13
+
14
+ @unittest.skipIf(torch is None, "Calibration tests require torch and safetensors")
15
+ class CalibrationTests(unittest.TestCase):
16
+ def test_fit(self):
17
+ z = torch.arange(-20, 21, dtype=torch.float32)[:, None]
18
+ gain, bias = fit_affine(z, z * 1.02 + 0.7, affine=True, ridge=0)
19
+ torch.testing.assert_close(gain, torch.tensor([1.02]))
20
+ torch.testing.assert_close(bias, torch.tensor([0.7]))
21
+ gain, _ = fit_affine(z, z * 2, affine=False)
22
+ torch.testing.assert_close(gain, torch.tensor([1.03]))
23
+
24
+ def test_zero(self):
25
+ z = torch.zeros(24, 8)
26
+ gain, bias = fit_affine(z, torch.ones_like(z), affine=True)
27
+ torch.testing.assert_close(gain, torch.ones(8))
28
+ self.assertTrue(torch.isfinite(bias).all())
29
+
30
+ def test_invalid_fit(self):
31
+ with self.assertRaises(ValueError):
32
+ fit_affine(torch.ones(2, 3), torch.ones(3, 2), affine=True)
33
+
34
+ def make_checkpoint(self, directory):
35
+ text = torch.nn.Module()
36
+ text.proj = torch.nn.Linear(1024, 1024, bias=False).to(torch.bfloat16).eval()
37
+ weights = torch.nn.Module()
38
+ weights.register_buffer("weight", torch.zeros(1024, 1024, dtype=torch.int8))
39
+ weights.register_buffer("weight_scale", torch.ones(1024))
40
+ weights.register_buffer("bias", torch.zeros(1024, dtype=torch.bfloat16))
41
+ writer = CalibrationWriter(directory, {"scope": "all"}, max_bytes=600_000)
42
+ writer.add("proj", weights)
43
+ return text, weights, writer.finish()
44
+
45
+ def test_indexed_roundtrip(self):
46
+ with tempfile.TemporaryDirectory() as tmp:
47
+ text, weights, path = self.make_checkpoint(tmp)
48
+ index = CalibrationIndex(path, text, "all")
49
+ loaded = index.load("proj", text.proj, backend="torch", tile=(128, 256), cluster=(2, 1))
50
+ for leaf in ("weight", "weight_scale", "bias"):
51
+ torch.testing.assert_close(getattr(loaded, leaf), getattr(weights, leaf), atol=0, rtol=0)
52
+ self.assertEqual(len(index.files), 2)
53
+ self.assertEqual(index.metadata["total_size"], 1024 * 1024 + 1024 * 6)
54
+
55
+ def test_reject_bad_index(self):
56
+ with tempfile.TemporaryDirectory() as tmp:
57
+ text, _, path = self.make_checkpoint(tmp)
58
+ original = json.loads(path.read_text())
59
+ for mutate in (
60
+ lambda d: d["metadata"].update(total_size=1),
61
+ lambda d: d["weight_map"].pop("proj.bias"),
62
+ lambda d: d["weight_map"].update({"proj.bias": "../escape.safetensors"}),
63
+ lambda d: d["weight_map"].update({"proj.bias": ["bad"]}),
64
+ lambda d: d["weight_map"].update({"proj.bias": "missing.safetensors"}),
65
+ lambda d: d["weight_map"].update({"proj.bias": d["weight_map"]["proj.weight"]}),
66
+ lambda d: d.update(metadata=[]),
67
+ ):
68
+ changed = json.loads(json.dumps(original))
69
+ mutate(changed)
70
+ path.write_text(json.dumps(changed))
71
+ with self.assertRaises(ValueError):
72
+ CalibrationIndex(path, text, "all")
73
+ path.write_text('{"metadata": {}, "metadata": {}, "weight_map": {}}')
74
+ with self.assertRaisesRegex(ValueError, "Duplicate"):
75
+ CalibrationIndex(path, text, "all")
76
+ path.write_text(json.dumps(original))
77
+ text.proj = torch.nn.Linear(1024, 2048).eval()
78
+ with self.assertRaisesRegex(ValueError, "shape/dtype"):
79
+ CalibrationIndex(path, text, "all")
80
+
81
+ def test_reject_invalid_values(self):
82
+ from safetensors.torch import save_file
83
+
84
+ with tempfile.TemporaryDirectory() as tmp:
85
+ text, weights, path = self.make_checkpoint(tmp)
86
+ index = CalibrationIndex(path, text, "all")
87
+ filename = index.weight_map["proj.weight_scale"]
88
+ for value in (0, float("nan")):
89
+ weights.weight_scale.fill_(value)
90
+ save_file({"proj.weight_scale": weights.weight_scale, "proj.bias": weights.bias}, index.files[filename])
91
+ index = CalibrationIndex(path, text, "all")
92
+ with self.assertRaisesRegex(ValueError, "Non-finite"):
93
+ index.load("proj", text.proj, backend="torch", tile=(128, 256), cluster=(2, 1))
94
+
95
+
96
+ if __name__ == "__main__":
97
+ unittest.main()
runtime/test_clef_text_metrics.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import unittest
2
+
3
+ from clef_text_metrics import decision_drift
4
+
5
+
6
+ class DecisionDriftTest(unittest.TestCase):
7
+ def test_exact_and_tie(self):
8
+ logits = [[[1.0, 1.0], [-3.0, -1.0, 2.0]]]
9
+ result = decision_drift(logits, logits)
10
+ self.assertEqual(result["decision_agreement"], 1)
11
+ self.assertEqual(result["logits_relative_l2"], 0)
12
+ self.assertEqual(result["reference_margin_min"], 0)
13
+ self.assertFalse(result["decisions"][0]["margin_exceeds_twice_error"])
14
+
15
+ def test_close_flip_is_reported(self):
16
+ result = decision_drift([[[1.0, 1.2], [5.1, 1.0]]], [[[1.1, 1.0], [5.0, 1.0]]])
17
+ self.assertEqual(result["decisions_matched"], 1)
18
+ self.assertEqual(result["decisions_total"], 2)
19
+ self.assertEqual(result["flips"][0]["question"], 0)
20
+ self.assertEqual(result["flips"][0]["candidate_choice"], 1)
21
+ self.assertTrue(result["decisions"][1]["margin_exceeds_twice_error"])
22
+ self.assertAlmostEqual(result["logits_max_abs"], 0.2)
23
+
24
+ def test_zero_reference(self):
25
+ result = decision_drift([[[0.0, 0.0]]], [[[0.0, 0.0]]])
26
+ self.assertEqual(result["logits_relative_l2"], 0)
27
+
28
+ def test_invalid_shapes_and_values(self):
29
+ for actual, expected in [([], []), ([], [[[1.0, 2.0]]]),
30
+ ([[[1.0]]], [[[1.0]]]), ([[[1.0, 2.0]]], [[[1.0, 2.0, 3.0]]]),
31
+ ([[[float("nan"), 2.0]]], [[[1.0, 2.0]]]),
32
+ ([[[1.0, 2.0]]], [[[float("inf"), 2.0]]])]:
33
+ with self.subTest(actual=actual, expected=expected), self.assertRaises(ValueError):
34
+ decision_drift(actual, expected)
35
+
36
+
37
+ if __name__ == "__main__":
38
+ unittest.main()
runtime/test_clef_text_profile.py ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import importlib.util
2
+ from pathlib import Path
3
+ from types import SimpleNamespace
4
+ import unittest
5
+ from unittest.mock import patch
6
+
7
+
8
+ spec = importlib.util.spec_from_file_location("profile_under_test", Path(__file__).with_name("clef_text_profile.py"))
9
+ profile = importlib.util.module_from_spec(spec)
10
+ with patch.dict("sys.modules", {"torch": SimpleNamespace(autograd=SimpleNamespace(DeviceType=SimpleNamespace(CUDA="cuda")))}):
11
+ spec.loader.exec_module(profile)
12
+
13
+
14
+ def event(name, milliseconds, device="cuda"):
15
+ return SimpleNamespace(name=name, device_type=device,
16
+ time_range=SimpleNamespace(elapsed_us=lambda: milliseconds * 1000))
17
+
18
+
19
+ class KernelSummaryTest(unittest.TestCase):
20
+ def test_leaf_kernels_are_not_double_counted_with_cpu_scopes(self):
21
+ events = [event("clef.mlp", 30, "cpu"), event("gemm", 2), event("gemm", 4),
22
+ event("convrot", 1), event("Memcpy HtoD", 0.5), event("Memset", 0.2)]
23
+ result = profile.summarize_kernels(SimpleNamespace(events=lambda: events), 2)
24
+ self.assertEqual(result["kernel_sum_ms"], 3.5)
25
+ self.assertEqual(result["kernel_count_per_forward"], 1.5)
26
+ self.assertEqual(result["kernels"][0]["name"], "gemm")
27
+ self.assertEqual(result["kernels"][0]["total_per_forward_ms"], 3)
28
+ self.assertEqual(result["kernels"][0]["median_call_ms"], 3)
29
+ self.assertEqual(len(result["transfers"]), 2)
30
+
31
+ def test_empty_capture(self):
32
+ result = profile.summarize_kernels(SimpleNamespace(events=lambda: []), 3)
33
+ self.assertEqual(result["kernel_sum_ms"], 0)
34
+ self.assertEqual(result["kernels"], [])
35
+
36
+ def test_reject_zero_iterations_before_touching_gpu(self):
37
+ with self.assertRaises(ValueError):
38
+ profile.profile_text(None, None, None, iterations=0)
39
+
40
+
41
+ if __name__ == "__main__":
42
+ unittest.main()