Publish calibrated CLEF INT8 ConvRot text-linear overrides
Browse files- LICENSE +202 -0
- NOTICE +20 -0
- README.md +181 -0
- SHA256SUMS +15 -0
- clef_text_int8_convrot_calibrated-00001-of-00014.safetensors +3 -0
- clef_text_int8_convrot_calibrated-00002-of-00014.safetensors +3 -0
- clef_text_int8_convrot_calibrated-00003-of-00014.safetensors +3 -0
- clef_text_int8_convrot_calibrated-00004-of-00014.safetensors +3 -0
- clef_text_int8_convrot_calibrated-00005-of-00014.safetensors +3 -0
- clef_text_int8_convrot_calibrated-00006-of-00014.safetensors +3 -0
- clef_text_int8_convrot_calibrated-00007-of-00014.safetensors +3 -0
- clef_text_int8_convrot_calibrated-00008-of-00014.safetensors +3 -0
- clef_text_int8_convrot_calibrated-00009-of-00014.safetensors +3 -0
- clef_text_int8_convrot_calibrated-00010-of-00014.safetensors +3 -0
- clef_text_int8_convrot_calibrated-00011-of-00014.safetensors +3 -0
- clef_text_int8_convrot_calibrated-00012-of-00014.safetensors +3 -0
- clef_text_int8_convrot_calibrated-00013-of-00014.safetensors +3 -0
- clef_text_int8_convrot_calibrated-00014-of-00014.safetensors +3 -0
- clef_text_int8_convrot_calibrated.safetensors.index.json +0 -0
- convrot_config.json +41 -0
- results/bracketed.json +1811 -0
- results/calibration.json +0 -0
- runtime/clef-bench.py +479 -0
- runtime/clef-bootstrap-runtime +44 -0
- runtime/clef-int8-bench.py +236 -0
- runtime/clef-int8-calibrate.py +215 -0
- runtime/clef-int8-calibration-check.py +56 -0
- runtime/clef_int8_calibration.py +275 -0
- runtime/clef_int8_cutlass.py +1307 -0
- runtime/clef_text_cute.py +341 -0
- runtime/clef_text_int8.py +154 -0
- runtime/clef_text_metrics.py +40 -0
- runtime/clef_text_profile.py +107 -0
- runtime/clef_vision_norm.py +217 -0
- runtime/test_clef_int8_calibration.py +97 -0
- runtime/test_clef_text_metrics.py +38 -0
- runtime/test_clef_text_profile.py +42 -0
LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright 2026 Alibaba Cloud
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
NOTICE
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CLEF Calibrated INT8 ConvRot Text Weights
|
| 2 |
+
|
| 3 |
+
Derived from Cloudflare/clef, revision
|
| 4 |
+
2f3de3dd85f379784083b0814d997ab627200f0c, released under Apache-2.0.
|
| 5 |
+
|
| 6 |
+
Modifications: selected large text projection weights were rotated and
|
| 7 |
+
quantized to INT8, calibrated using per-output-row clipping and bounded
|
| 8 |
+
scale/bias fitting, and exported as indexed safetensors linear overrides.
|
| 9 |
+
Other model components are not included in this distribution.
|
| 10 |
+
|
| 11 |
+
The CuTe GEMM implementation in runtime/clef_int8_cutlass.py is adapted from
|
| 12 |
+
NVIDIA CUTLASS examples and retains its full BSD-3-Clause notice:
|
| 13 |
+
Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
| 14 |
+
|
| 15 |
+
Comfy Kitchen supplies the activation ConvRot quantizer as an external
|
| 16 |
+
dependency. Comfy Kitchen, PyTorch, Transformers, Flash Linear Attention,
|
| 17 |
+
CUTLASS, and other dependencies retain their respective licenses.
|
| 18 |
+
|
| 19 |
+
This derivative is not an official Cloudflare release and is not endorsed
|
| 20 |
+
by Cloudflare or NVIDIA.
|
README.md
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
base_model: Cloudflare/clef
|
| 4 |
+
base_model_relation: quantized
|
| 5 |
+
language:
|
| 6 |
+
- en
|
| 7 |
+
tags:
|
| 8 |
+
- clef
|
| 9 |
+
- int8
|
| 10 |
+
- convrot
|
| 11 |
+
- cute-dsl
|
| 12 |
+
- custom-code
|
| 13 |
+
- experimental
|
| 14 |
+
---
|
| 15 |
+
|
| 16 |
+
# CLEF Calibrated INT8 ConvRot Text Weights
|
| 17 |
+
|
| 18 |
+
Experimental calibrated W8A8 ConvRot text-linear overrides for
|
| 19 |
+
[Cloudflare/clef](https://huggingface.co/Cloudflare/clef/tree/2f3de3dd85f379784083b0814d997ab627200f0c).
|
| 20 |
+
The CuTeDSL H100 runtime applies dequantization scales and calibrated bias in
|
| 21 |
+
the GEMM epilogue, with no additional inference kernels.
|
| 22 |
+
|
| 23 |
+
**This is not a standalone Transformers checkpoint.** Load the original CLEF
|
| 24 |
+
release at revision `2f3de3dd85f379784083b0814d997ab627200f0c`, then apply these
|
| 25 |
+
overrides using the included runtime. `AutoModel.from_pretrained()` cannot load
|
| 26 |
+
this repository directly. Tokenizer, processor, embeddings, norms, small control
|
| 27 |
+
projections, joint decision head, and vision weights remain in the original BF16
|
| 28 |
+
checkpoint and are not included here.
|
| 29 |
+
|
| 30 |
+
## Contents
|
| 31 |
+
|
| 32 |
+
- 400 calibrated large text projections: 192 MLP, 144 linear-attention, and 64
|
| 33 |
+
full-attention projections.
|
| 34 |
+
- 1,200 tensors: rotated INT8 weights, FP32 per-output-row scales, and BF16 bias.
|
| 35 |
+
- 14 zero-padded safetensors shards targeting 1.9 GB each; 24,350,359,552
|
| 36 |
+
uncompressed tensor bytes in total (22.68 GiB).
|
| 37 |
+
- `clef_text_int8_convrot_calibrated.safetensors.index.json`: complete tensor map,
|
| 38 |
+
calibration provenance, source fingerprint, and byte count.
|
| 39 |
+
- `SHA256SUMS`: checksums for the shards and index.
|
| 40 |
+
- `runtime/`: loader, CuTe kernels, calibration tools, benchmark helpers, and tests.
|
| 41 |
+
- `results/`: calibration and independent bracketed benchmark reports.
|
| 42 |
+
- `convrot_config.json`: format, source revision, runtime, and calibration settings.
|
| 43 |
+
|
| 44 |
+
The format is `clef-convrot-linear-overrides-v1`, with ConvRot group size 256.
|
| 45 |
+
Each projection stores `weight` shaped `[out_features, in_features]`, and
|
| 46 |
+
`weight_scale` and `bias` shaped `[out_features]`. Activation rotation and
|
| 47 |
+
row-wise INT8 quantization use the unchanged Comfy Kitchen ConvRot quantizer.
|
| 48 |
+
|
| 49 |
+
## Calibration And Evaluation
|
| 50 |
+
|
| 51 |
+
BF16 activations were collected on the pinned real CLEF text backbone. Training
|
| 52 |
+
used 12 synthetic records, with 16 sampled token rows per record per projection
|
| 53 |
+
(192 rows). A disjoint six-record selection set provided 96 rows per projection.
|
| 54 |
+
Per-output-row candidates use rotated-weight absolute-max clipping fractions
|
| 55 |
+
`[1.0, 0.99, 0.97, 0.94, 0.90]`, a scale gain bounded to three percent around one,
|
| 56 |
+
and optional bias correction. The ridge coefficient is 0.05. A candidate must
|
| 57 |
+
reduce selection MSE by at least 0.5%; unchanged weights are retained otherwise.
|
| 58 |
+
This is clipping and scale/bias fitting, not GPTQ or AdaRound.
|
| 59 |
+
|
| 60 |
+
Evaluation used 16 independent synthetic held-out records plus seven older
|
| 61 |
+
regression probes. Agreement is with BF16 decisions, not ground-truth task
|
| 62 |
+
accuracy. Both INT8 variants matched 48/48 held-out decisions. Older regression
|
| 63 |
+
agreement increased from 30/32 to 31/32, for an overall change from 78/80 to 79/80.
|
| 64 |
+
One disagreement remains at 1,278 tokens. Calibration did not improve every record.
|
| 65 |
+
|
| 66 |
+
| Metric | Uncalibrated INT8 | Calibrated INT8 |
|
| 67 |
+
| --- | ---: | ---: |
|
| 68 |
+
| Mean held-out relative logit L2 | 0.02166 | 0.01837 |
|
| 69 |
+
| Mean held-out relative hidden-state L2 | 0.10379 | 0.09733 |
|
| 70 |
+
| BF16 decision agreement, all 23 records | 78/80 | 79/80 |
|
| 71 |
+
| Warm median inference, 1,502 tokens | 121.17 ms | 121.10 ms |
|
| 72 |
+
| Resident GPU memory | 28.59 GiB | 28.59 GiB |
|
| 73 |
+
|
| 74 |
+
Mean held-out relative logit drift decreased by 15.2% and hidden-state drift by
|
| 75 |
+
6.2%. An independent 30-repeat benchmark measured 120.89 ms calibrated versus
|
| 76 |
+
139.88 ms bracketed fused BF16 (1.157x). Shared/unshared calibrated paths matched
|
| 77 |
+
exactly. At 298-312 tokens, this INT8 runtime was slower than fused BF16; the
|
| 78 |
+
speedup is workload-dependent.
|
| 79 |
+
|
| 80 |
+
Indexed reload produced exactly identical hidden states and logits on all 23
|
| 81 |
+
evaluation records. All 13 unit tests passed. The calibrated GPU smoke check
|
| 82 |
+
also passed integer-reference epilogue parity, non-default CUDA streams, and
|
| 83 |
+
changed-input CUDA graph replay.
|
| 84 |
+
|
| 85 |
+
**Synthetic calibration only.** These small sets do not establish production
|
| 86 |
+
accuracy, probability calibration, or image/video quality. Vision weights were
|
| 87 |
+
not quantized, and multimodal quality of this calibrated text path was not
|
| 88 |
+
evaluated. Validate with representative records before deployment. This release
|
| 89 |
+
does not claim to reproduce the upstream Decision Index scores.
|
| 90 |
+
|
| 91 |
+
## Runtime
|
| 92 |
+
|
| 93 |
+
Measured on NVIDIA H100 80 GB, SM90, in `nvcr.io/nvidia/pytorch:25.09-py3`:
|
| 94 |
+
|
| 95 |
+
- PyTorch `2.9.0a0+50eac811a6.nv25.09`, CUDA 13.
|
| 96 |
+
- Transformers `5.12.1`.
|
| 97 |
+
- NVIDIA CUTLASS DSL `4.8.0` and Apache TVM FFI `0.1.14.post1`.
|
| 98 |
+
- Comfy Kitchen `0.2.31`.
|
| 99 |
+
- Flash Linear Attention `0.5.2` and causal-conv1d `1.7.0`.
|
| 100 |
+
|
| 101 |
+
The CuTe backend requires inference mode, BF16 input activations, and SM90.
|
| 102 |
+
The measured GEMM tile is `[128, 256]` and cluster is `[2, 1]`. Recheck numerical
|
| 103 |
+
parity when changing hardware or library versions. A Torch integer-reference
|
| 104 |
+
backend is available, but the reported performance uses CuTe.
|
| 105 |
+
|
| 106 |
+
The bundled `runtime/clef-bootstrap-runtime` bootstraps dependencies in the
|
| 107 |
+
NVIDIA container. Set `CLEF_INSTALL_INT8=1` when invoking it; it is a Linux/CUDA
|
| 108 |
+
bootstrap script, not a CPU or macOS installer.
|
| 109 |
+
|
| 110 |
+
## Loading
|
| 111 |
+
|
| 112 |
+
After installing the runtime dependencies on an H100:
|
| 113 |
+
|
| 114 |
+
```python
|
| 115 |
+
import json
|
| 116 |
+
import sys
|
| 117 |
+
from pathlib import Path
|
| 118 |
+
|
| 119 |
+
import torch
|
| 120 |
+
from huggingface_hub import snapshot_download
|
| 121 |
+
|
| 122 |
+
source = Path(snapshot_download(
|
| 123 |
+
"Cloudflare/clef",
|
| 124 |
+
revision="2f3de3dd85f379784083b0814d997ab627200f0c",
|
| 125 |
+
))
|
| 126 |
+
overrides = Path(snapshot_download("terminusresearch/cloudflare-clef-int8-convrot"))
|
| 127 |
+
sys.path.insert(0, str(source))
|
| 128 |
+
sys.path.insert(0, str(overrides / "runtime"))
|
| 129 |
+
|
| 130 |
+
from joint_schema_model import load_release_model, encode_record, collate_records
|
| 131 |
+
from clef_int8_calibration import source_digest
|
| 132 |
+
from clef_text_int8 import quantize_text
|
| 133 |
+
from clef_text_cute import patch_text
|
| 134 |
+
|
| 135 |
+
index = overrides / "clef_text_int8_convrot_calibrated.safetensors.index.json"
|
| 136 |
+
if json.loads(index.read_text())["metadata"]["source_digest"] != source_digest(source):
|
| 137 |
+
raise ValueError("Source config/code/index does not match this calibrated release")
|
| 138 |
+
|
| 139 |
+
with torch.inference_mode():
|
| 140 |
+
model, processor = load_release_model(source, device="cuda", dtype=torch.bfloat16)
|
| 141 |
+
base = (model.language_model.get_base_model()
|
| 142 |
+
if hasattr(model.language_model, "get_base_model") else model.language_model)
|
| 143 |
+
text = getattr(base.model, "language_model", base.model)
|
| 144 |
+
quantize_text(text, scope="all", cluster=(2, 1), calibration_index=index)
|
| 145 |
+
restore = patch_text(text, gva=True, gates=True, heads=True,
|
| 146 |
+
share_mlp_quant=True, share_attn_quant=True)
|
| 147 |
+
|
| 148 |
+
record = {
|
| 149 |
+
"state": {"summary": "The customer confirms service is restored."},
|
| 150 |
+
"questions": {
|
| 151 |
+
"route": {
|
| 152 |
+
"type": "choice",
|
| 153 |
+
"instructions": "Choose the appropriate handling route.",
|
| 154 |
+
"criteria": {"close": "Resolved issue", "review": "Unresolved issue"},
|
| 155 |
+
},
|
| 156 |
+
},
|
| 157 |
+
}
|
| 158 |
+
encoded = encode_record(processor.tokenizer, record, processor=processor)
|
| 159 |
+
batch = collate_records([encoded], processor.tokenizer.pad_token_id, torch.device("cuda"))
|
| 160 |
+
logits = model(batch)[0]
|
| 161 |
+
probabilities = [question_logits.float().softmax(-1).tolist() for question_logits in logits]
|
| 162 |
+
print(probabilities)
|
| 163 |
+
```
|
| 164 |
+
|
| 165 |
+
The loader validates coverage, safe shard paths, tensor shapes/dtypes, scales,
|
| 166 |
+
duplicate keys, and the uncompressed byte total, and streams tensors from the
|
| 167 |
+
indexed shards. No weight quantization runs when loading the overrides. The
|
| 168 |
+
current benchmark loader first loads the original BF16 model, so startup needs
|
| 169 |
+
enough memory for that original model; the 28.59 GiB figure is post-conversion
|
| 170 |
+
residency, not the cold-load peak. This release is an inference-only overlay,
|
| 171 |
+
not a training checkpoint or a prepackaged Cog image.
|
| 172 |
+
|
| 173 |
+
## License
|
| 174 |
+
|
| 175 |
+
The calibrated weights derive from the pinned Apache-2.0
|
| 176 |
+
[Cloudflare/clef release](https://huggingface.co/Cloudflare/clef/tree/2f3de3dd85f379784083b0814d997ab627200f0c).
|
| 177 |
+
`LICENSE` preserves the upstream license. `runtime/clef_int8_cutlass.py` includes
|
| 178 |
+
NVIDIA's BSD-3-Clause copyright notice and terms, which remain applicable to
|
| 179 |
+
that kernel. See `NOTICE` for modification and attribution details. This is an
|
| 180 |
+
experimental derivative published by Terminus Research, not an official
|
| 181 |
+
Cloudflare release.
|
SHA256SUMS
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
747660618289ecec786a7f87859c438bd9c19d7837f38c51d062540b72829bb3 clef_text_int8_convrot_calibrated-00001-of-00014.safetensors
|
| 2 |
+
6bc4ad5220a3baa029a986048c14e822fd3380434df805e0e999273e49f85f35 clef_text_int8_convrot_calibrated-00002-of-00014.safetensors
|
| 3 |
+
c413e03e53138766b18404f60835f1f2dd386036e7557e1ffe655c568f2f6ded clef_text_int8_convrot_calibrated-00003-of-00014.safetensors
|
| 4 |
+
b9bd15b2bce887e9f06c42d750e661b3b1ecf4e17f645761e9d4dd77ea0d45a8 clef_text_int8_convrot_calibrated-00004-of-00014.safetensors
|
| 5 |
+
b24d00380e98f9b3ecede33d569c214a87183516221c0e7f7b76c19e16948972 clef_text_int8_convrot_calibrated-00005-of-00014.safetensors
|
| 6 |
+
17fafd4fb52cfd432133243f81dfdb37558869d8cda06ddd9cb8405676799a80 clef_text_int8_convrot_calibrated-00006-of-00014.safetensors
|
| 7 |
+
ee706c7a99464f48940a31a1662eb980d8503bf1593c4116e0fccd5d2c1aa70c clef_text_int8_convrot_calibrated-00007-of-00014.safetensors
|
| 8 |
+
2086b697130268f2252bf3303ceeb86639da97887b3593120f2f73592631d5b7 clef_text_int8_convrot_calibrated-00008-of-00014.safetensors
|
| 9 |
+
454ec8aff5104c987d45870f2daee3f0728387cd3318235302c7fee24e5c37b2 clef_text_int8_convrot_calibrated-00009-of-00014.safetensors
|
| 10 |
+
450d935a79b08ba51e84873f502be9626eb33714e46aac57f5072662ee106c90 clef_text_int8_convrot_calibrated-00010-of-00014.safetensors
|
| 11 |
+
68961a270992e855dc5907e8204b2fba582b9e8f31e57003c283ddb1a2a419d7 clef_text_int8_convrot_calibrated-00011-of-00014.safetensors
|
| 12 |
+
ee8ab2ec02071177cd794907ed2f040951e3041db6ebe706fa1b6503441822ac clef_text_int8_convrot_calibrated-00012-of-00014.safetensors
|
| 13 |
+
6a66131491b7a1ce89f6383b40c010253de0ec891c901fbb37cccfa319eebac2 clef_text_int8_convrot_calibrated-00013-of-00014.safetensors
|
| 14 |
+
3fc9d98b968495ba0db5bf2a7da735edb329c7dd1ce2f455189c24c99975c52e clef_text_int8_convrot_calibrated-00014-of-00014.safetensors
|
| 15 |
+
c078d31e8f16c9425f34dbda14b041abe425ac66bdfe9cfa745dbf3353497494 clef_text_int8_convrot_calibrated.safetensors.index.json
|
clef_text_int8_convrot_calibrated-00001-of-00014.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:747660618289ecec786a7f87859c438bd9c19d7837f38c51d062540b72829bb3
|
| 3 |
+
size 1815846088
|
clef_text_int8_convrot_calibrated-00002-of-00014.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6bc4ad5220a3baa029a986048c14e822fd3380434df805e0e999273e49f85f35
|
| 3 |
+
size 1815772368
|
clef_text_int8_convrot_calibrated-00003-of-00014.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c413e03e53138766b18404f60835f1f2dd386036e7557e1ffe655c568f2f6ded
|
| 3 |
+
size 1815772440
|
clef_text_int8_convrot_calibrated-00004-of-00014.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b9bd15b2bce887e9f06c42d750e661b3b1ecf4e17f645761e9d4dd77ea0d45a8
|
| 3 |
+
size 1894508392
|
clef_text_int8_convrot_calibrated-00005-of-00014.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b24d00380e98f9b3ecede33d569c214a87183516221c0e7f7b76c19e16948972
|
| 3 |
+
size 1873511704
|
clef_text_int8_convrot_calibrated-00006-of-00014.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:17fafd4fb52cfd432133243f81dfdb37558869d8cda06ddd9cb8405676799a80
|
| 3 |
+
size 1852515592
|
clef_text_int8_convrot_calibrated-00007-of-00014.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ee706c7a99464f48940a31a1662eb980d8503bf1593c4116e0fccd5d2c1aa70c
|
| 3 |
+
size 1873517848
|
clef_text_int8_convrot_calibrated-00008-of-00014.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2086b697130268f2252bf3303ceeb86639da97887b3593120f2f73592631d5b7
|
| 3 |
+
size 1815846176
|
clef_text_int8_convrot_calibrated-00009-of-00014.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:454ec8aff5104c987d45870f2daee3f0728387cd3318235302c7fee24e5c37b2
|
| 3 |
+
size 1894508384
|
clef_text_int8_convrot_calibrated-00010-of-00014.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:450d935a79b08ba51e84873f502be9626eb33714e46aac57f5072662ee106c90
|
| 3 |
+
size 1815772456
|
clef_text_int8_convrot_calibrated-00011-of-00014.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:68961a270992e855dc5907e8204b2fba582b9e8f31e57003c283ddb1a2a419d7
|
| 3 |
+
size 1815772456
|
clef_text_int8_convrot_calibrated-00012-of-00014.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ee8ab2ec02071177cd794907ed2f040951e3041db6ebe706fa1b6503441822ac
|
| 3 |
+
size 1873511696
|
clef_text_int8_convrot_calibrated-00013-of-00014.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6a66131491b7a1ce89f6383b40c010253de0ec891c901fbb37cccfa319eebac2
|
| 3 |
+
size 1894514544
|
clef_text_int8_convrot_calibrated-00014-of-00014.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3fc9d98b968495ba0db5bf2a7da735edb329c7dd1ce2f455189c24c99975c52e
|
| 3 |
+
size 299115768
|
clef_text_int8_convrot_calibrated.safetensors.index.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
convrot_config.json
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "clef-convrot-linear-overrides-v1",
|
| 3 |
+
"standalone_model": false,
|
| 4 |
+
"source_model": "Cloudflare/clef",
|
| 5 |
+
"source_revision": "2f3de3dd85f379784083b0814d997ab627200f0c",
|
| 6 |
+
"source_digest": "cae974bb5b4b21095342ecb5dcee32771d65a4bade3367334a762e1ae390f96e",
|
| 7 |
+
"scope": "all",
|
| 8 |
+
"group_size": 256,
|
| 9 |
+
"scheme": "W8A8 ConvRot",
|
| 10 |
+
"index": "clef_text_int8_convrot_calibrated.safetensors.index.json",
|
| 11 |
+
"linears": 400,
|
| 12 |
+
"tensors": 1200,
|
| 13 |
+
"total_size": 24350359552,
|
| 14 |
+
"calibration_data": "synthetic",
|
| 15 |
+
"train_records": 12,
|
| 16 |
+
"selection_records": 6,
|
| 17 |
+
"heldout_records": 16,
|
| 18 |
+
"sample_rows_per_record": 16,
|
| 19 |
+
"clips": [
|
| 20 |
+
1.0,
|
| 21 |
+
0.99,
|
| 22 |
+
0.97,
|
| 23 |
+
0.94,
|
| 24 |
+
0.9
|
| 25 |
+
],
|
| 26 |
+
"ridge": 0.05,
|
| 27 |
+
"max_gain": 0.03,
|
| 28 |
+
"min_improvement": 0.005,
|
| 29 |
+
"tile": [
|
| 30 |
+
128,
|
| 31 |
+
256
|
| 32 |
+
],
|
| 33 |
+
"cluster": [
|
| 34 |
+
2,
|
| 35 |
+
1
|
| 36 |
+
],
|
| 37 |
+
"cute_device_capability": [
|
| 38 |
+
9,
|
| 39 |
+
0
|
| 40 |
+
]
|
| 41 |
+
}
|
results/bracketed.json
ADDED
|
@@ -0,0 +1,1811 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_path": "Cloudflare/clef@2f3de3dd85f379784083b0814d997ab627200f0c",
|
| 3 |
+
"torch": "2.9.0a0+50eac811a6.nv25.09",
|
| 4 |
+
"device": "NVIDIA H100 80GB HBM3",
|
| 5 |
+
"backend": "cute",
|
| 6 |
+
"heads": true,
|
| 7 |
+
"share_mlp_input": true,
|
| 8 |
+
"share_attn_input": true,
|
| 9 |
+
"modes": {
|
| 10 |
+
"bf16_before": {
|
| 11 |
+
"cases": [
|
| 12 |
+
{
|
| 13 |
+
"case": 0,
|
| 14 |
+
"tokens": 1278,
|
| 15 |
+
"total_ms": {
|
| 16 |
+
"mean": 120.7420423006018,
|
| 17 |
+
"median": 120.32128451392055,
|
| 18 |
+
"p95": 126.36434938758612,
|
| 19 |
+
"min": 117.37496964633465,
|
| 20 |
+
"max": 128.86804901063442
|
| 21 |
+
},
|
| 22 |
+
"backbone_ms": {
|
| 23 |
+
"mean": 117.27207411701481,
|
| 24 |
+
"median": 117.1717680990696,
|
| 25 |
+
"p95": 118.6508983373642,
|
| 26 |
+
"min": 115.35093002021313,
|
| 27 |
+
"max": 118.73763613402843
|
| 28 |
+
},
|
| 29 |
+
"hidden_error": {
|
| 30 |
+
"max_abs": 0.0,
|
| 31 |
+
"relative_l2": 0.0
|
| 32 |
+
},
|
| 33 |
+
"drift": {
|
| 34 |
+
"logits_max_abs": 0.0,
|
| 35 |
+
"logits_relative_l2": 0.0,
|
| 36 |
+
"decisions_matched": 8,
|
| 37 |
+
"decisions_total": 8,
|
| 38 |
+
"decision_agreement": 1.0,
|
| 39 |
+
"reference_margin_min": 0.0625,
|
| 40 |
+
"reference_margin_median": 0.25,
|
| 41 |
+
"flips": [],
|
| 42 |
+
"decisions": [
|
| 43 |
+
{
|
| 44 |
+
"record": 0,
|
| 45 |
+
"question": 0,
|
| 46 |
+
"reference_choice": 0,
|
| 47 |
+
"candidate_choice": 0,
|
| 48 |
+
"matched": true,
|
| 49 |
+
"reference_margin": 0.3671875,
|
| 50 |
+
"logits_max_abs": 0.0,
|
| 51 |
+
"margin_exceeds_twice_error": true
|
| 52 |
+
},
|
| 53 |
+
{
|
| 54 |
+
"record": 0,
|
| 55 |
+
"question": 1,
|
| 56 |
+
"reference_choice": 0,
|
| 57 |
+
"candidate_choice": 0,
|
| 58 |
+
"matched": true,
|
| 59 |
+
"reference_margin": 0.2265625,
|
| 60 |
+
"logits_max_abs": 0.0,
|
| 61 |
+
"margin_exceeds_twice_error": true
|
| 62 |
+
},
|
| 63 |
+
{
|
| 64 |
+
"record": 0,
|
| 65 |
+
"question": 2,
|
| 66 |
+
"reference_choice": 0,
|
| 67 |
+
"candidate_choice": 0,
|
| 68 |
+
"matched": true,
|
| 69 |
+
"reference_margin": 0.2265625,
|
| 70 |
+
"logits_max_abs": 0.0,
|
| 71 |
+
"margin_exceeds_twice_error": true
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"record": 0,
|
| 75 |
+
"question": 3,
|
| 76 |
+
"reference_choice": 0,
|
| 77 |
+
"candidate_choice": 0,
|
| 78 |
+
"matched": true,
|
| 79 |
+
"reference_margin": 0.2734375,
|
| 80 |
+
"logits_max_abs": 0.0,
|
| 81 |
+
"margin_exceeds_twice_error": true
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"record": 0,
|
| 85 |
+
"question": 4,
|
| 86 |
+
"reference_choice": 0,
|
| 87 |
+
"candidate_choice": 0,
|
| 88 |
+
"matched": true,
|
| 89 |
+
"reference_margin": 0.419921875,
|
| 90 |
+
"logits_max_abs": 0.0,
|
| 91 |
+
"margin_exceeds_twice_error": true
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"record": 0,
|
| 95 |
+
"question": 5,
|
| 96 |
+
"reference_choice": 0,
|
| 97 |
+
"candidate_choice": 0,
|
| 98 |
+
"matched": true,
|
| 99 |
+
"reference_margin": 0.37109375,
|
| 100 |
+
"logits_max_abs": 0.0,
|
| 101 |
+
"margin_exceeds_twice_error": true
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"record": 0,
|
| 105 |
+
"question": 6,
|
| 106 |
+
"reference_choice": 0,
|
| 107 |
+
"candidate_choice": 0,
|
| 108 |
+
"matched": true,
|
| 109 |
+
"reference_margin": 0.0625,
|
| 110 |
+
"logits_max_abs": 0.0,
|
| 111 |
+
"margin_exceeds_twice_error": true
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"record": 0,
|
| 115 |
+
"question": 7,
|
| 116 |
+
"reference_choice": 3,
|
| 117 |
+
"candidate_choice": 3,
|
| 118 |
+
"matched": true,
|
| 119 |
+
"reference_margin": 0.1328125,
|
| 120 |
+
"logits_max_abs": 0.0,
|
| 121 |
+
"margin_exceeds_twice_error": true
|
| 122 |
+
}
|
| 123 |
+
]
|
| 124 |
+
},
|
| 125 |
+
"resident_vram_bytes": 55004415488,
|
| 126 |
+
"peak_vram_bytes": 55225683456
|
| 127 |
+
},
|
| 128 |
+
{
|
| 129 |
+
"case": 1,
|
| 130 |
+
"tokens": 1502,
|
| 131 |
+
"total_ms": {
|
| 132 |
+
"mean": 140.4689157071213,
|
| 133 |
+
"median": 139.46003885939717,
|
| 134 |
+
"p95": 146.0675811395049,
|
| 135 |
+
"min": 136.4539759233594,
|
| 136 |
+
"max": 155.13416472822428
|
| 137 |
+
},
|
| 138 |
+
"backbone_ms": {
|
| 139 |
+
"mean": 136.78044530873498,
|
| 140 |
+
"median": 136.8254255503416,
|
| 141 |
+
"p95": 139.53122403472662,
|
| 142 |
+
"min": 133.93584918230772,
|
| 143 |
+
"max": 140.33847115933895
|
| 144 |
+
},
|
| 145 |
+
"hidden_error": {
|
| 146 |
+
"max_abs": 0.0,
|
| 147 |
+
"relative_l2": 0.0
|
| 148 |
+
},
|
| 149 |
+
"drift": {
|
| 150 |
+
"logits_max_abs": 0.0,
|
| 151 |
+
"logits_relative_l2": 0.0,
|
| 152 |
+
"decisions_matched": 8,
|
| 153 |
+
"decisions_total": 8,
|
| 154 |
+
"decision_agreement": 1.0,
|
| 155 |
+
"reference_margin_min": 0.25,
|
| 156 |
+
"reference_margin_median": 0.42578125,
|
| 157 |
+
"flips": [],
|
| 158 |
+
"decisions": [
|
| 159 |
+
{
|
| 160 |
+
"record": 0,
|
| 161 |
+
"question": 0,
|
| 162 |
+
"reference_choice": 0,
|
| 163 |
+
"candidate_choice": 0,
|
| 164 |
+
"matched": true,
|
| 165 |
+
"reference_margin": 0.69921875,
|
| 166 |
+
"logits_max_abs": 0.0,
|
| 167 |
+
"margin_exceeds_twice_error": true
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"record": 0,
|
| 171 |
+
"question": 1,
|
| 172 |
+
"reference_choice": 0,
|
| 173 |
+
"candidate_choice": 0,
|
| 174 |
+
"matched": true,
|
| 175 |
+
"reference_margin": 0.392578125,
|
| 176 |
+
"logits_max_abs": 0.0,
|
| 177 |
+
"margin_exceeds_twice_error": true
|
| 178 |
+
},
|
| 179 |
+
{
|
| 180 |
+
"record": 0,
|
| 181 |
+
"question": 2,
|
| 182 |
+
"reference_choice": 0,
|
| 183 |
+
"candidate_choice": 0,
|
| 184 |
+
"matched": true,
|
| 185 |
+
"reference_margin": 0.25,
|
| 186 |
+
"logits_max_abs": 0.0,
|
| 187 |
+
"margin_exceeds_twice_error": true
|
| 188 |
+
},
|
| 189 |
+
{
|
| 190 |
+
"record": 0,
|
| 191 |
+
"question": 3,
|
| 192 |
+
"reference_choice": 0,
|
| 193 |
+
"candidate_choice": 0,
|
| 194 |
+
"matched": true,
|
| 195 |
+
"reference_margin": 0.48828125,
|
| 196 |
+
"logits_max_abs": 0.0,
|
| 197 |
+
"margin_exceeds_twice_error": true
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"record": 0,
|
| 201 |
+
"question": 4,
|
| 202 |
+
"reference_choice": 0,
|
| 203 |
+
"candidate_choice": 0,
|
| 204 |
+
"matched": true,
|
| 205 |
+
"reference_margin": 0.58984375,
|
| 206 |
+
"logits_max_abs": 0.0,
|
| 207 |
+
"margin_exceeds_twice_error": true
|
| 208 |
+
},
|
| 209 |
+
{
|
| 210 |
+
"record": 0,
|
| 211 |
+
"question": 5,
|
| 212 |
+
"reference_choice": 0,
|
| 213 |
+
"candidate_choice": 0,
|
| 214 |
+
"matched": true,
|
| 215 |
+
"reference_margin": 0.4296875,
|
| 216 |
+
"logits_max_abs": 0.0,
|
| 217 |
+
"margin_exceeds_twice_error": true
|
| 218 |
+
},
|
| 219 |
+
{
|
| 220 |
+
"record": 0,
|
| 221 |
+
"question": 6,
|
| 222 |
+
"reference_choice": 0,
|
| 223 |
+
"candidate_choice": 0,
|
| 224 |
+
"matched": true,
|
| 225 |
+
"reference_margin": 0.28515625,
|
| 226 |
+
"logits_max_abs": 0.0,
|
| 227 |
+
"margin_exceeds_twice_error": true
|
| 228 |
+
},
|
| 229 |
+
{
|
| 230 |
+
"record": 0,
|
| 231 |
+
"question": 7,
|
| 232 |
+
"reference_choice": 0,
|
| 233 |
+
"candidate_choice": 0,
|
| 234 |
+
"matched": true,
|
| 235 |
+
"reference_margin": 0.421875,
|
| 236 |
+
"logits_max_abs": 0.0,
|
| 237 |
+
"margin_exceeds_twice_error": true
|
| 238 |
+
}
|
| 239 |
+
]
|
| 240 |
+
},
|
| 241 |
+
"resident_vram_bytes": 55004415488,
|
| 242 |
+
"peak_vram_bytes": 55267193344
|
| 243 |
+
},
|
| 244 |
+
{
|
| 245 |
+
"case": 2,
|
| 246 |
+
"tokens": 2270,
|
| 247 |
+
"total_ms": {
|
| 248 |
+
"mean": 207.17171784490347,
|
| 249 |
+
"median": 206.8792046047747,
|
| 250 |
+
"p95": 215.44742956757545,
|
| 251 |
+
"min": 200.07592719048262,
|
| 252 |
+
"max": 220.3005338087678
|
| 253 |
+
},
|
| 254 |
+
"backbone_ms": {
|
| 255 |
+
"mean": 202.72988236198822,
|
| 256 |
+
"median": 202.4515587836504,
|
| 257 |
+
"p95": 205.48917073756456,
|
| 258 |
+
"min": 199.79984499514103,
|
| 259 |
+
"max": 208.22311658412218
|
| 260 |
+
},
|
| 261 |
+
"hidden_error": {
|
| 262 |
+
"max_abs": 0.0,
|
| 263 |
+
"relative_l2": 0.0
|
| 264 |
+
},
|
| 265 |
+
"drift": {
|
| 266 |
+
"logits_max_abs": 0.0,
|
| 267 |
+
"logits_relative_l2": 0.0,
|
| 268 |
+
"decisions_matched": 8,
|
| 269 |
+
"decisions_total": 8,
|
| 270 |
+
"decision_agreement": 1.0,
|
| 271 |
+
"reference_margin_min": 0.01953125,
|
| 272 |
+
"reference_margin_median": 0.12109375,
|
| 273 |
+
"flips": [],
|
| 274 |
+
"decisions": [
|
| 275 |
+
{
|
| 276 |
+
"record": 0,
|
| 277 |
+
"question": 0,
|
| 278 |
+
"reference_choice": 0,
|
| 279 |
+
"candidate_choice": 0,
|
| 280 |
+
"matched": true,
|
| 281 |
+
"reference_margin": 0.4765625,
|
| 282 |
+
"logits_max_abs": 0.0,
|
| 283 |
+
"margin_exceeds_twice_error": true
|
| 284 |
+
},
|
| 285 |
+
{
|
| 286 |
+
"record": 0,
|
| 287 |
+
"question": 1,
|
| 288 |
+
"reference_choice": 0,
|
| 289 |
+
"candidate_choice": 0,
|
| 290 |
+
"matched": true,
|
| 291 |
+
"reference_margin": 0.306640625,
|
| 292 |
+
"logits_max_abs": 0.0,
|
| 293 |
+
"margin_exceeds_twice_error": true
|
| 294 |
+
},
|
| 295 |
+
{
|
| 296 |
+
"record": 0,
|
| 297 |
+
"question": 2,
|
| 298 |
+
"reference_choice": 0,
|
| 299 |
+
"candidate_choice": 0,
|
| 300 |
+
"matched": true,
|
| 301 |
+
"reference_margin": 0.08203125,
|
| 302 |
+
"logits_max_abs": 0.0,
|
| 303 |
+
"margin_exceeds_twice_error": true
|
| 304 |
+
},
|
| 305 |
+
{
|
| 306 |
+
"record": 0,
|
| 307 |
+
"question": 3,
|
| 308 |
+
"reference_choice": 0,
|
| 309 |
+
"candidate_choice": 0,
|
| 310 |
+
"matched": true,
|
| 311 |
+
"reference_margin": 0.1171875,
|
| 312 |
+
"logits_max_abs": 0.0,
|
| 313 |
+
"margin_exceeds_twice_error": true
|
| 314 |
+
},
|
| 315 |
+
{
|
| 316 |
+
"record": 0,
|
| 317 |
+
"question": 4,
|
| 318 |
+
"reference_choice": 0,
|
| 319 |
+
"candidate_choice": 0,
|
| 320 |
+
"matched": true,
|
| 321 |
+
"reference_margin": 0.01953125,
|
| 322 |
+
"logits_max_abs": 0.0,
|
| 323 |
+
"margin_exceeds_twice_error": true
|
| 324 |
+
},
|
| 325 |
+
{
|
| 326 |
+
"record": 0,
|
| 327 |
+
"question": 5,
|
| 328 |
+
"reference_choice": 0,
|
| 329 |
+
"candidate_choice": 0,
|
| 330 |
+
"matched": true,
|
| 331 |
+
"reference_margin": 0.05859375,
|
| 332 |
+
"logits_max_abs": 0.0,
|
| 333 |
+
"margin_exceeds_twice_error": true
|
| 334 |
+
},
|
| 335 |
+
{
|
| 336 |
+
"record": 0,
|
| 337 |
+
"question": 6,
|
| 338 |
+
"reference_choice": 3,
|
| 339 |
+
"candidate_choice": 3,
|
| 340 |
+
"matched": true,
|
| 341 |
+
"reference_margin": 0.125,
|
| 342 |
+
"logits_max_abs": 0.0,
|
| 343 |
+
"margin_exceeds_twice_error": true
|
| 344 |
+
},
|
| 345 |
+
{
|
| 346 |
+
"record": 0,
|
| 347 |
+
"question": 7,
|
| 348 |
+
"reference_choice": 0,
|
| 349 |
+
"candidate_choice": 0,
|
| 350 |
+
"matched": true,
|
| 351 |
+
"reference_margin": 0.265625,
|
| 352 |
+
"logits_max_abs": 0.0,
|
| 353 |
+
"margin_exceeds_twice_error": true
|
| 354 |
+
}
|
| 355 |
+
]
|
| 356 |
+
},
|
| 357 |
+
"resident_vram_bytes": 55004415488,
|
| 358 |
+
"peak_vram_bytes": 55398181376
|
| 359 |
+
},
|
| 360 |
+
{
|
| 361 |
+
"case": 3,
|
| 362 |
+
"tokens": 298,
|
| 363 |
+
"total_ms": {
|
| 364 |
+
"mean": 66.36064605166516,
|
| 365 |
+
"median": 66.18425762280822,
|
| 366 |
+
"p95": 67.18876399099827,
|
| 367 |
+
"min": 65.79676549881697,
|
| 368 |
+
"max": 70.03371138125658
|
| 369 |
+
},
|
| 370 |
+
"backbone_ms": {
|
| 371 |
+
"mean": 63.170733861625195,
|
| 372 |
+
"median": 62.95886728912592,
|
| 373 |
+
"p95": 64.23373054713011,
|
| 374 |
+
"min": 62.67668306827545,
|
| 375 |
+
"max": 64.84911683946848
|
| 376 |
+
},
|
| 377 |
+
"hidden_error": {
|
| 378 |
+
"max_abs": 0.0,
|
| 379 |
+
"relative_l2": 0.0
|
| 380 |
+
},
|
| 381 |
+
"drift": {
|
| 382 |
+
"logits_max_abs": 0.0,
|
| 383 |
+
"logits_relative_l2": 0.0,
|
| 384 |
+
"decisions_matched": 2,
|
| 385 |
+
"decisions_total": 2,
|
| 386 |
+
"decision_agreement": 1.0,
|
| 387 |
+
"reference_margin_min": 4.31640625,
|
| 388 |
+
"reference_margin_median": 4.353515625,
|
| 389 |
+
"flips": [],
|
| 390 |
+
"decisions": [
|
| 391 |
+
{
|
| 392 |
+
"record": 0,
|
| 393 |
+
"question": 0,
|
| 394 |
+
"reference_choice": 3,
|
| 395 |
+
"candidate_choice": 3,
|
| 396 |
+
"matched": true,
|
| 397 |
+
"reference_margin": 4.390625,
|
| 398 |
+
"logits_max_abs": 0.0,
|
| 399 |
+
"margin_exceeds_twice_error": true
|
| 400 |
+
},
|
| 401 |
+
{
|
| 402 |
+
"record": 0,
|
| 403 |
+
"question": 1,
|
| 404 |
+
"reference_choice": 1,
|
| 405 |
+
"candidate_choice": 1,
|
| 406 |
+
"matched": true,
|
| 407 |
+
"reference_margin": 4.31640625,
|
| 408 |
+
"logits_max_abs": 0.0,
|
| 409 |
+
"margin_exceeds_twice_error": true
|
| 410 |
+
}
|
| 411 |
+
]
|
| 412 |
+
},
|
| 413 |
+
"resident_vram_bytes": 55004415488,
|
| 414 |
+
"peak_vram_bytes": 55057560576
|
| 415 |
+
},
|
| 416 |
+
{
|
| 417 |
+
"case": 4,
|
| 418 |
+
"tokens": 303,
|
| 419 |
+
"total_ms": {
|
| 420 |
+
"mean": 66.08876610795657,
|
| 421 |
+
"median": 66.01279973983765,
|
| 422 |
+
"p95": 66.44898280501366,
|
| 423 |
+
"min": 65.63741154968739,
|
| 424 |
+
"max": 67.82077439129353
|
| 425 |
+
},
|
| 426 |
+
"backbone_ms": {
|
| 427 |
+
"mean": 62.835137452930205,
|
| 428 |
+
"median": 62.80959444120526,
|
| 429 |
+
"p95": 63.14793787896633,
|
| 430 |
+
"min": 62.608824111521244,
|
| 431 |
+
"max": 63.26063070446253
|
| 432 |
+
},
|
| 433 |
+
"hidden_error": {
|
| 434 |
+
"max_abs": 0.0,
|
| 435 |
+
"relative_l2": 0.0
|
| 436 |
+
},
|
| 437 |
+
"drift": {
|
| 438 |
+
"logits_max_abs": 0.0,
|
| 439 |
+
"logits_relative_l2": 0.0,
|
| 440 |
+
"decisions_matched": 2,
|
| 441 |
+
"decisions_total": 2,
|
| 442 |
+
"decision_agreement": 1.0,
|
| 443 |
+
"reference_margin_min": 2.8984375,
|
| 444 |
+
"reference_margin_median": 3.73046875,
|
| 445 |
+
"flips": [],
|
| 446 |
+
"decisions": [
|
| 447 |
+
{
|
| 448 |
+
"record": 0,
|
| 449 |
+
"question": 0,
|
| 450 |
+
"reference_choice": 2,
|
| 451 |
+
"candidate_choice": 2,
|
| 452 |
+
"matched": true,
|
| 453 |
+
"reference_margin": 4.5625,
|
| 454 |
+
"logits_max_abs": 0.0,
|
| 455 |
+
"margin_exceeds_twice_error": true
|
| 456 |
+
},
|
| 457 |
+
{
|
| 458 |
+
"record": 0,
|
| 459 |
+
"question": 1,
|
| 460 |
+
"reference_choice": 0,
|
| 461 |
+
"candidate_choice": 0,
|
| 462 |
+
"matched": true,
|
| 463 |
+
"reference_margin": 2.8984375,
|
| 464 |
+
"logits_max_abs": 0.0,
|
| 465 |
+
"margin_exceeds_twice_error": true
|
| 466 |
+
}
|
| 467 |
+
]
|
| 468 |
+
},
|
| 469 |
+
"resident_vram_bytes": 55004415488,
|
| 470 |
+
"peak_vram_bytes": 55058505216
|
| 471 |
+
},
|
| 472 |
+
{
|
| 473 |
+
"case": 5,
|
| 474 |
+
"tokens": 304,
|
| 475 |
+
"total_ms": {
|
| 476 |
+
"mean": 66.5853524270157,
|
| 477 |
+
"median": 66.38480117544532,
|
| 478 |
+
"p95": 68.45066975802183,
|
| 479 |
+
"min": 65.95136970281601,
|
| 480 |
+
"max": 69.19064838439226
|
| 481 |
+
},
|
| 482 |
+
"backbone_ms": {
|
| 483 |
+
"mean": 63.43956707666318,
|
| 484 |
+
"median": 63.35389753803611,
|
| 485 |
+
"p95": 64.41042199730873,
|
| 486 |
+
"min": 62.95953691005707,
|
| 487 |
+
"max": 65.36273565143347
|
| 488 |
+
},
|
| 489 |
+
"hidden_error": {
|
| 490 |
+
"max_abs": 0.0,
|
| 491 |
+
"relative_l2": 0.0
|
| 492 |
+
},
|
| 493 |
+
"drift": {
|
| 494 |
+
"logits_max_abs": 0.0,
|
| 495 |
+
"logits_relative_l2": 0.0,
|
| 496 |
+
"decisions_matched": 2,
|
| 497 |
+
"decisions_total": 2,
|
| 498 |
+
"decision_agreement": 1.0,
|
| 499 |
+
"reference_margin_min": 4.2109375,
|
| 500 |
+
"reference_margin_median": 4.4228515625,
|
| 501 |
+
"flips": [],
|
| 502 |
+
"decisions": [
|
| 503 |
+
{
|
| 504 |
+
"record": 0,
|
| 505 |
+
"question": 0,
|
| 506 |
+
"reference_choice": 0,
|
| 507 |
+
"candidate_choice": 0,
|
| 508 |
+
"matched": true,
|
| 509 |
+
"reference_margin": 4.2109375,
|
| 510 |
+
"logits_max_abs": 0.0,
|
| 511 |
+
"margin_exceeds_twice_error": true
|
| 512 |
+
},
|
| 513 |
+
{
|
| 514 |
+
"record": 0,
|
| 515 |
+
"question": 1,
|
| 516 |
+
"reference_choice": 1,
|
| 517 |
+
"candidate_choice": 1,
|
| 518 |
+
"matched": true,
|
| 519 |
+
"reference_margin": 4.634765625,
|
| 520 |
+
"logits_max_abs": 0.0,
|
| 521 |
+
"margin_exceeds_twice_error": true
|
| 522 |
+
}
|
| 523 |
+
]
|
| 524 |
+
},
|
| 525 |
+
"resident_vram_bytes": 55004415488,
|
| 526 |
+
"peak_vram_bytes": 55058544128
|
| 527 |
+
},
|
| 528 |
+
{
|
| 529 |
+
"case": 6,
|
| 530 |
+
"tokens": 312,
|
| 531 |
+
"total_ms": {
|
| 532 |
+
"mean": 66.48833425715566,
|
| 533 |
+
"median": 66.47943751886487,
|
| 534 |
+
"p95": 66.86687096953392,
|
| 535 |
+
"min": 66.1541111767292,
|
| 536 |
+
"max": 66.8799402192235
|
| 537 |
+
},
|
| 538 |
+
"backbone_ms": {
|
| 539 |
+
"mean": 63.53455794354281,
|
| 540 |
+
"median": 63.32046305760741,
|
| 541 |
+
"p95": 63.97620867937803,
|
| 542 |
+
"min": 63.109404407441616,
|
| 543 |
+
"max": 68.88721697032452
|
| 544 |
+
},
|
| 545 |
+
"hidden_error": {
|
| 546 |
+
"max_abs": 0.0,
|
| 547 |
+
"relative_l2": 0.0
|
| 548 |
+
},
|
| 549 |
+
"drift": {
|
| 550 |
+
"logits_max_abs": 0.0,
|
| 551 |
+
"logits_relative_l2": 0.0,
|
| 552 |
+
"decisions_matched": 2,
|
| 553 |
+
"decisions_total": 2,
|
| 554 |
+
"decision_agreement": 1.0,
|
| 555 |
+
"reference_margin_min": 1.7109375,
|
| 556 |
+
"reference_margin_median": 2.55859375,
|
| 557 |
+
"flips": [],
|
| 558 |
+
"decisions": [
|
| 559 |
+
{
|
| 560 |
+
"record": 0,
|
| 561 |
+
"question": 0,
|
| 562 |
+
"reference_choice": 2,
|
| 563 |
+
"candidate_choice": 2,
|
| 564 |
+
"matched": true,
|
| 565 |
+
"reference_margin": 3.40625,
|
| 566 |
+
"logits_max_abs": 0.0,
|
| 567 |
+
"margin_exceeds_twice_error": true
|
| 568 |
+
},
|
| 569 |
+
{
|
| 570 |
+
"record": 0,
|
| 571 |
+
"question": 1,
|
| 572 |
+
"reference_choice": 0,
|
| 573 |
+
"candidate_choice": 0,
|
| 574 |
+
"matched": true,
|
| 575 |
+
"reference_margin": 1.7109375,
|
| 576 |
+
"logits_max_abs": 0.0,
|
| 577 |
+
"margin_exceeds_twice_error": true
|
| 578 |
+
}
|
| 579 |
+
]
|
| 580 |
+
},
|
| 581 |
+
"resident_vram_bytes": 55004415488,
|
| 582 |
+
"peak_vram_bytes": 55060240896
|
| 583 |
+
}
|
| 584 |
+
],
|
| 585 |
+
"conversion": null,
|
| 586 |
+
"shared_input_exact": null,
|
| 587 |
+
"patch_statistics": {
|
| 588 |
+
"rms_calls": 43456,
|
| 589 |
+
"swiglu_calls": 28672,
|
| 590 |
+
"residual_calls": 28672,
|
| 591 |
+
"delta_calls": 21504,
|
| 592 |
+
"shared_quant_calls": 0,
|
| 593 |
+
"shared_attn_quant_calls": 0,
|
| 594 |
+
"fallbacks": 0
|
| 595 |
+
}
|
| 596 |
+
},
|
| 597 |
+
"all": {
|
| 598 |
+
"cases": [
|
| 599 |
+
{
|
| 600 |
+
"case": 0,
|
| 601 |
+
"tokens": 1278,
|
| 602 |
+
"total_ms": {
|
| 603 |
+
"mean": 107.05213258042932,
|
| 604 |
+
"median": 107.02321119606495,
|
| 605 |
+
"p95": 107.38053172826767,
|
| 606 |
+
"min": 106.93768132477999,
|
| 607 |
+
"max": 107.4241129681468
|
| 608 |
+
},
|
| 609 |
+
"backbone_ms": {
|
| 610 |
+
"mean": 101.29548273980618,
|
| 611 |
+
"median": 101.28400847315788,
|
| 612 |
+
"p95": 101.40760242938995,
|
| 613 |
+
"min": 101.21874045580626,
|
| 614 |
+
"max": 101.45805310457945
|
| 615 |
+
},
|
| 616 |
+
"hidden_error": {
|
| 617 |
+
"max_abs": 30.8125,
|
| 618 |
+
"relative_l2": 0.13932070136070251
|
| 619 |
+
},
|
| 620 |
+
"drift": {
|
| 621 |
+
"logits_max_abs": 0.234375,
|
| 622 |
+
"logits_relative_l2": 0.24432059284457605,
|
| 623 |
+
"decisions_matched": 7,
|
| 624 |
+
"decisions_total": 8,
|
| 625 |
+
"decision_agreement": 0.875,
|
| 626 |
+
"reference_margin_min": 0.0625,
|
| 627 |
+
"reference_margin_median": 0.25,
|
| 628 |
+
"flips": [
|
| 629 |
+
{
|
| 630 |
+
"record": 0,
|
| 631 |
+
"question": 7,
|
| 632 |
+
"reference_choice": 3,
|
| 633 |
+
"candidate_choice": 0,
|
| 634 |
+
"matched": false,
|
| 635 |
+
"reference_margin": 0.1328125,
|
| 636 |
+
"logits_max_abs": 0.234375,
|
| 637 |
+
"margin_exceeds_twice_error": false
|
| 638 |
+
}
|
| 639 |
+
],
|
| 640 |
+
"decisions": [
|
| 641 |
+
{
|
| 642 |
+
"record": 0,
|
| 643 |
+
"question": 0,
|
| 644 |
+
"reference_choice": 0,
|
| 645 |
+
"candidate_choice": 0,
|
| 646 |
+
"matched": true,
|
| 647 |
+
"reference_margin": 0.3671875,
|
| 648 |
+
"logits_max_abs": 0.15625,
|
| 649 |
+
"margin_exceeds_twice_error": true
|
| 650 |
+
},
|
| 651 |
+
{
|
| 652 |
+
"record": 0,
|
| 653 |
+
"question": 1,
|
| 654 |
+
"reference_choice": 0,
|
| 655 |
+
"candidate_choice": 0,
|
| 656 |
+
"matched": true,
|
| 657 |
+
"reference_margin": 0.2265625,
|
| 658 |
+
"logits_max_abs": 0.109375,
|
| 659 |
+
"margin_exceeds_twice_error": true
|
| 660 |
+
},
|
| 661 |
+
{
|
| 662 |
+
"record": 0,
|
| 663 |
+
"question": 2,
|
| 664 |
+
"reference_choice": 0,
|
| 665 |
+
"candidate_choice": 0,
|
| 666 |
+
"matched": true,
|
| 667 |
+
"reference_margin": 0.2265625,
|
| 668 |
+
"logits_max_abs": 0.080078125,
|
| 669 |
+
"margin_exceeds_twice_error": true
|
| 670 |
+
},
|
| 671 |
+
{
|
| 672 |
+
"record": 0,
|
| 673 |
+
"question": 3,
|
| 674 |
+
"reference_choice": 0,
|
| 675 |
+
"candidate_choice": 0,
|
| 676 |
+
"matched": true,
|
| 677 |
+
"reference_margin": 0.2734375,
|
| 678 |
+
"logits_max_abs": 0.07421875,
|
| 679 |
+
"margin_exceeds_twice_error": true
|
| 680 |
+
},
|
| 681 |
+
{
|
| 682 |
+
"record": 0,
|
| 683 |
+
"question": 4,
|
| 684 |
+
"reference_choice": 0,
|
| 685 |
+
"candidate_choice": 0,
|
| 686 |
+
"matched": true,
|
| 687 |
+
"reference_margin": 0.419921875,
|
| 688 |
+
"logits_max_abs": 0.119140625,
|
| 689 |
+
"margin_exceeds_twice_error": true
|
| 690 |
+
},
|
| 691 |
+
{
|
| 692 |
+
"record": 0,
|
| 693 |
+
"question": 5,
|
| 694 |
+
"reference_choice": 0,
|
| 695 |
+
"candidate_choice": 0,
|
| 696 |
+
"matched": true,
|
| 697 |
+
"reference_margin": 0.37109375,
|
| 698 |
+
"logits_max_abs": 0.109375,
|
| 699 |
+
"margin_exceeds_twice_error": true
|
| 700 |
+
},
|
| 701 |
+
{
|
| 702 |
+
"record": 0,
|
| 703 |
+
"question": 6,
|
| 704 |
+
"reference_choice": 0,
|
| 705 |
+
"candidate_choice": 0,
|
| 706 |
+
"matched": true,
|
| 707 |
+
"reference_margin": 0.0625,
|
| 708 |
+
"logits_max_abs": 0.099609375,
|
| 709 |
+
"margin_exceeds_twice_error": false
|
| 710 |
+
},
|
| 711 |
+
{
|
| 712 |
+
"record": 0,
|
| 713 |
+
"question": 7,
|
| 714 |
+
"reference_choice": 3,
|
| 715 |
+
"candidate_choice": 0,
|
| 716 |
+
"matched": false,
|
| 717 |
+
"reference_margin": 0.1328125,
|
| 718 |
+
"logits_max_abs": 0.234375,
|
| 719 |
+
"margin_exceeds_twice_error": false
|
| 720 |
+
}
|
| 721 |
+
]
|
| 722 |
+
},
|
| 723 |
+
"resident_vram_bytes": 30701897216,
|
| 724 |
+
"peak_vram_bytes": 30936492544,
|
| 725 |
+
"speedup_vs_fused_bf16": 1.125855436584257
|
| 726 |
+
},
|
| 727 |
+
{
|
| 728 |
+
"case": 1,
|
| 729 |
+
"tokens": 1502,
|
| 730 |
+
"total_ms": {
|
| 731 |
+
"mean": 120.8846031067272,
|
| 732 |
+
"median": 120.88794028386474,
|
| 733 |
+
"p95": 121.01801671087742,
|
| 734 |
+
"min": 120.7155492156744,
|
| 735 |
+
"max": 121.02974485605955
|
| 736 |
+
},
|
| 737 |
+
"backbone_ms": {
|
| 738 |
+
"mean": 115.24689175809424,
|
| 739 |
+
"median": 115.23000663146377,
|
| 740 |
+
"p95": 115.4045844450593,
|
| 741 |
+
"min": 115.11011142283678,
|
| 742 |
+
"max": 115.42476527392864
|
| 743 |
+
},
|
| 744 |
+
"hidden_error": {
|
| 745 |
+
"max_abs": 31.283203125,
|
| 746 |
+
"relative_l2": 0.2085031121969223
|
| 747 |
+
},
|
| 748 |
+
"drift": {
|
| 749 |
+
"logits_max_abs": 0.29296875,
|
| 750 |
+
"logits_relative_l2": 0.5195755740972049,
|
| 751 |
+
"decisions_matched": 8,
|
| 752 |
+
"decisions_total": 8,
|
| 753 |
+
"decision_agreement": 1.0,
|
| 754 |
+
"reference_margin_min": 0.25,
|
| 755 |
+
"reference_margin_median": 0.42578125,
|
| 756 |
+
"flips": [],
|
| 757 |
+
"decisions": [
|
| 758 |
+
{
|
| 759 |
+
"record": 0,
|
| 760 |
+
"question": 0,
|
| 761 |
+
"reference_choice": 0,
|
| 762 |
+
"candidate_choice": 0,
|
| 763 |
+
"matched": true,
|
| 764 |
+
"reference_margin": 0.69921875,
|
| 765 |
+
"logits_max_abs": 0.29296875,
|
| 766 |
+
"margin_exceeds_twice_error": true
|
| 767 |
+
},
|
| 768 |
+
{
|
| 769 |
+
"record": 0,
|
| 770 |
+
"question": 1,
|
| 771 |
+
"reference_choice": 0,
|
| 772 |
+
"candidate_choice": 0,
|
| 773 |
+
"matched": true,
|
| 774 |
+
"reference_margin": 0.392578125,
|
| 775 |
+
"logits_max_abs": 0.267578125,
|
| 776 |
+
"margin_exceeds_twice_error": false
|
| 777 |
+
},
|
| 778 |
+
{
|
| 779 |
+
"record": 0,
|
| 780 |
+
"question": 2,
|
| 781 |
+
"reference_choice": 0,
|
| 782 |
+
"candidate_choice": 0,
|
| 783 |
+
"matched": true,
|
| 784 |
+
"reference_margin": 0.25,
|
| 785 |
+
"logits_max_abs": 0.26171875,
|
| 786 |
+
"margin_exceeds_twice_error": false
|
| 787 |
+
},
|
| 788 |
+
{
|
| 789 |
+
"record": 0,
|
| 790 |
+
"question": 3,
|
| 791 |
+
"reference_choice": 0,
|
| 792 |
+
"candidate_choice": 0,
|
| 793 |
+
"matched": true,
|
| 794 |
+
"reference_margin": 0.48828125,
|
| 795 |
+
"logits_max_abs": 0.24609375,
|
| 796 |
+
"margin_exceeds_twice_error": false
|
| 797 |
+
},
|
| 798 |
+
{
|
| 799 |
+
"record": 0,
|
| 800 |
+
"question": 4,
|
| 801 |
+
"reference_choice": 0,
|
| 802 |
+
"candidate_choice": 0,
|
| 803 |
+
"matched": true,
|
| 804 |
+
"reference_margin": 0.58984375,
|
| 805 |
+
"logits_max_abs": 0.2734375,
|
| 806 |
+
"margin_exceeds_twice_error": true
|
| 807 |
+
},
|
| 808 |
+
{
|
| 809 |
+
"record": 0,
|
| 810 |
+
"question": 5,
|
| 811 |
+
"reference_choice": 0,
|
| 812 |
+
"candidate_choice": 0,
|
| 813 |
+
"matched": true,
|
| 814 |
+
"reference_margin": 0.4296875,
|
| 815 |
+
"logits_max_abs": 0.23828125,
|
| 816 |
+
"margin_exceeds_twice_error": false
|
| 817 |
+
},
|
| 818 |
+
{
|
| 819 |
+
"record": 0,
|
| 820 |
+
"question": 6,
|
| 821 |
+
"reference_choice": 0,
|
| 822 |
+
"candidate_choice": 0,
|
| 823 |
+
"matched": true,
|
| 824 |
+
"reference_margin": 0.28515625,
|
| 825 |
+
"logits_max_abs": 0.28125,
|
| 826 |
+
"margin_exceeds_twice_error": false
|
| 827 |
+
},
|
| 828 |
+
{
|
| 829 |
+
"record": 0,
|
| 830 |
+
"question": 7,
|
| 831 |
+
"reference_choice": 0,
|
| 832 |
+
"candidate_choice": 0,
|
| 833 |
+
"matched": true,
|
| 834 |
+
"reference_margin": 0.421875,
|
| 835 |
+
"logits_max_abs": 0.22265625,
|
| 836 |
+
"margin_exceeds_twice_error": false
|
| 837 |
+
}
|
| 838 |
+
]
|
| 839 |
+
},
|
| 840 |
+
"resident_vram_bytes": 30701897216,
|
| 841 |
+
"peak_vram_bytes": 30979767808,
|
| 842 |
+
"speedup_vs_fused_bf16": 1.1570775125081916
|
| 843 |
+
},
|
| 844 |
+
{
|
| 845 |
+
"case": 2,
|
| 846 |
+
"tokens": 2270,
|
| 847 |
+
"total_ms": {
|
| 848 |
+
"mean": 172.77655890211463,
|
| 849 |
+
"median": 172.73904010653496,
|
| 850 |
+
"p95": 172.9419371113181,
|
| 851 |
+
"min": 172.47775755822659,
|
| 852 |
+
"max": 174.69287477433681
|
| 853 |
+
},
|
| 854 |
+
"backbone_ms": {
|
| 855 |
+
"mean": 166.9653277223309,
|
| 856 |
+
"median": 166.96942318230867,
|
| 857 |
+
"p95": 167.18766652047634,
|
| 858 |
+
"min": 166.6247695684433,
|
| 859 |
+
"max": 167.29615069925785
|
| 860 |
+
},
|
| 861 |
+
"hidden_error": {
|
| 862 |
+
"max_abs": 45.296875,
|
| 863 |
+
"relative_l2": 0.2817922830581665
|
| 864 |
+
},
|
| 865 |
+
"drift": {
|
| 866 |
+
"logits_max_abs": 0.21484375,
|
| 867 |
+
"logits_relative_l2": 0.3134054938237255,
|
| 868 |
+
"decisions_matched": 8,
|
| 869 |
+
"decisions_total": 8,
|
| 870 |
+
"decision_agreement": 1.0,
|
| 871 |
+
"reference_margin_min": 0.01953125,
|
| 872 |
+
"reference_margin_median": 0.12109375,
|
| 873 |
+
"flips": [],
|
| 874 |
+
"decisions": [
|
| 875 |
+
{
|
| 876 |
+
"record": 0,
|
| 877 |
+
"question": 0,
|
| 878 |
+
"reference_choice": 0,
|
| 879 |
+
"candidate_choice": 0,
|
| 880 |
+
"matched": true,
|
| 881 |
+
"reference_margin": 0.4765625,
|
| 882 |
+
"logits_max_abs": 0.1484375,
|
| 883 |
+
"margin_exceeds_twice_error": true
|
| 884 |
+
},
|
| 885 |
+
{
|
| 886 |
+
"record": 0,
|
| 887 |
+
"question": 1,
|
| 888 |
+
"reference_choice": 0,
|
| 889 |
+
"candidate_choice": 0,
|
| 890 |
+
"matched": true,
|
| 891 |
+
"reference_margin": 0.306640625,
|
| 892 |
+
"logits_max_abs": 0.111328125,
|
| 893 |
+
"margin_exceeds_twice_error": true
|
| 894 |
+
},
|
| 895 |
+
{
|
| 896 |
+
"record": 0,
|
| 897 |
+
"question": 2,
|
| 898 |
+
"reference_choice": 0,
|
| 899 |
+
"candidate_choice": 0,
|
| 900 |
+
"matched": true,
|
| 901 |
+
"reference_margin": 0.08203125,
|
| 902 |
+
"logits_max_abs": 0.1484375,
|
| 903 |
+
"margin_exceeds_twice_error": false
|
| 904 |
+
},
|
| 905 |
+
{
|
| 906 |
+
"record": 0,
|
| 907 |
+
"question": 3,
|
| 908 |
+
"reference_choice": 0,
|
| 909 |
+
"candidate_choice": 0,
|
| 910 |
+
"matched": true,
|
| 911 |
+
"reference_margin": 0.1171875,
|
| 912 |
+
"logits_max_abs": 0.1171875,
|
| 913 |
+
"margin_exceeds_twice_error": false
|
| 914 |
+
},
|
| 915 |
+
{
|
| 916 |
+
"record": 0,
|
| 917 |
+
"question": 4,
|
| 918 |
+
"reference_choice": 0,
|
| 919 |
+
"candidate_choice": 0,
|
| 920 |
+
"matched": true,
|
| 921 |
+
"reference_margin": 0.01953125,
|
| 922 |
+
"logits_max_abs": 0.13671875,
|
| 923 |
+
"margin_exceeds_twice_error": false
|
| 924 |
+
},
|
| 925 |
+
{
|
| 926 |
+
"record": 0,
|
| 927 |
+
"question": 5,
|
| 928 |
+
"reference_choice": 0,
|
| 929 |
+
"candidate_choice": 0,
|
| 930 |
+
"matched": true,
|
| 931 |
+
"reference_margin": 0.05859375,
|
| 932 |
+
"logits_max_abs": 0.08984375,
|
| 933 |
+
"margin_exceeds_twice_error": false
|
| 934 |
+
},
|
| 935 |
+
{
|
| 936 |
+
"record": 0,
|
| 937 |
+
"question": 6,
|
| 938 |
+
"reference_choice": 3,
|
| 939 |
+
"candidate_choice": 3,
|
| 940 |
+
"matched": true,
|
| 941 |
+
"reference_margin": 0.125,
|
| 942 |
+
"logits_max_abs": 0.15625,
|
| 943 |
+
"margin_exceeds_twice_error": false
|
| 944 |
+
},
|
| 945 |
+
{
|
| 946 |
+
"record": 0,
|
| 947 |
+
"question": 7,
|
| 948 |
+
"reference_choice": 0,
|
| 949 |
+
"candidate_choice": 0,
|
| 950 |
+
"matched": true,
|
| 951 |
+
"reference_margin": 0.265625,
|
| 952 |
+
"logits_max_abs": 0.21484375,
|
| 953 |
+
"margin_exceeds_twice_error": false
|
| 954 |
+
}
|
| 955 |
+
]
|
| 956 |
+
},
|
| 957 |
+
"resident_vram_bytes": 30701897216,
|
| 958 |
+
"peak_vram_bytes": 31118588416,
|
| 959 |
+
"speedup_vs_fused_bf16": 1.1976097894375464
|
| 960 |
+
},
|
| 961 |
+
{
|
| 962 |
+
"case": 3,
|
| 963 |
+
"tokens": 298,
|
| 964 |
+
"total_ms": {
|
| 965 |
+
"mean": 82.499930759271,
|
| 966 |
+
"median": 82.49658718705177,
|
| 967 |
+
"p95": 82.73806516081095,
|
| 968 |
+
"min": 82.13112596422434,
|
| 969 |
+
"max": 82.77752809226513
|
| 970 |
+
},
|
| 971 |
+
"backbone_ms": {
|
| 972 |
+
"mean": 79.38881159449616,
|
| 973 |
+
"median": 79.29326640442014,
|
| 974 |
+
"p95": 80.56880813091993,
|
| 975 |
+
"min": 78.97979486733675,
|
| 976 |
+
"max": 80.72016015648842
|
| 977 |
+
},
|
| 978 |
+
"hidden_error": {
|
| 979 |
+
"max_abs": 8.25,
|
| 980 |
+
"relative_l2": 0.08705292642116547
|
| 981 |
+
},
|
| 982 |
+
"drift": {
|
| 983 |
+
"logits_max_abs": 0.171875,
|
| 984 |
+
"logits_relative_l2": 0.027874615994997076,
|
| 985 |
+
"decisions_matched": 2,
|
| 986 |
+
"decisions_total": 2,
|
| 987 |
+
"decision_agreement": 1.0,
|
| 988 |
+
"reference_margin_min": 4.31640625,
|
| 989 |
+
"reference_margin_median": 4.353515625,
|
| 990 |
+
"flips": [],
|
| 991 |
+
"decisions": [
|
| 992 |
+
{
|
| 993 |
+
"record": 0,
|
| 994 |
+
"question": 0,
|
| 995 |
+
"reference_choice": 3,
|
| 996 |
+
"candidate_choice": 3,
|
| 997 |
+
"matched": true,
|
| 998 |
+
"reference_margin": 4.390625,
|
| 999 |
+
"logits_max_abs": 0.0625,
|
| 1000 |
+
"margin_exceeds_twice_error": true
|
| 1001 |
+
},
|
| 1002 |
+
{
|
| 1003 |
+
"record": 0,
|
| 1004 |
+
"question": 1,
|
| 1005 |
+
"reference_choice": 1,
|
| 1006 |
+
"candidate_choice": 1,
|
| 1007 |
+
"matched": true,
|
| 1008 |
+
"reference_margin": 4.31640625,
|
| 1009 |
+
"logits_max_abs": 0.171875,
|
| 1010 |
+
"margin_exceeds_twice_error": true
|
| 1011 |
+
}
|
| 1012 |
+
]
|
| 1013 |
+
},
|
| 1014 |
+
"resident_vram_bytes": 30701897216,
|
| 1015 |
+
"peak_vram_bytes": 30758282240,
|
| 1016 |
+
"speedup_vs_fused_bf16": 0.7940032277701942
|
| 1017 |
+
},
|
| 1018 |
+
{
|
| 1019 |
+
"case": 4,
|
| 1020 |
+
"tokens": 303,
|
| 1021 |
+
"total_ms": {
|
| 1022 |
+
"mean": 83.24504966537158,
|
| 1023 |
+
"median": 82.82628236338496,
|
| 1024 |
+
"p95": 87.72443886846304,
|
| 1025 |
+
"min": 82.26131554692984,
|
| 1026 |
+
"max": 87.76593301445246
|
| 1027 |
+
},
|
| 1028 |
+
"backbone_ms": {
|
| 1029 |
+
"mean": 79.43275828535359,
|
| 1030 |
+
"median": 79.20759683474898,
|
| 1031 |
+
"p95": 79.86426446586847,
|
| 1032 |
+
"min": 78.9574459195137,
|
| 1033 |
+
"max": 83.8590394705534
|
| 1034 |
+
},
|
| 1035 |
+
"hidden_error": {
|
| 1036 |
+
"max_abs": 10.75,
|
| 1037 |
+
"relative_l2": 0.0930745080113411
|
| 1038 |
+
},
|
| 1039 |
+
"drift": {
|
| 1040 |
+
"logits_max_abs": 0.09375,
|
| 1041 |
+
"logits_relative_l2": 0.012005244385097194,
|
| 1042 |
+
"decisions_matched": 2,
|
| 1043 |
+
"decisions_total": 2,
|
| 1044 |
+
"decision_agreement": 1.0,
|
| 1045 |
+
"reference_margin_min": 2.8984375,
|
| 1046 |
+
"reference_margin_median": 3.73046875,
|
| 1047 |
+
"flips": [],
|
| 1048 |
+
"decisions": [
|
| 1049 |
+
{
|
| 1050 |
+
"record": 0,
|
| 1051 |
+
"question": 0,
|
| 1052 |
+
"reference_choice": 2,
|
| 1053 |
+
"candidate_choice": 2,
|
| 1054 |
+
"matched": true,
|
| 1055 |
+
"reference_margin": 4.5625,
|
| 1056 |
+
"logits_max_abs": 0.09375,
|
| 1057 |
+
"margin_exceeds_twice_error": true
|
| 1058 |
+
},
|
| 1059 |
+
{
|
| 1060 |
+
"record": 0,
|
| 1061 |
+
"question": 1,
|
| 1062 |
+
"reference_choice": 0,
|
| 1063 |
+
"candidate_choice": 0,
|
| 1064 |
+
"matched": true,
|
| 1065 |
+
"reference_margin": 2.8984375,
|
| 1066 |
+
"logits_max_abs": 0.0078125,
|
| 1067 |
+
"margin_exceeds_twice_error": true
|
| 1068 |
+
}
|
| 1069 |
+
]
|
| 1070 |
+
},
|
| 1071 |
+
"resident_vram_bytes": 30701897216,
|
| 1072 |
+
"peak_vram_bytes": 30759723520,
|
| 1073 |
+
"speedup_vs_fused_bf16": 0.7882684823613131
|
| 1074 |
+
},
|
| 1075 |
+
{
|
| 1076 |
+
"case": 5,
|
| 1077 |
+
"tokens": 304,
|
| 1078 |
+
"total_ms": {
|
| 1079 |
+
"mean": 82.33912341917554,
|
| 1080 |
+
"median": 82.14510511606932,
|
| 1081 |
+
"p95": 84.59722064435482,
|
| 1082 |
+
"min": 81.95048198103905,
|
| 1083 |
+
"max": 85.46646125614643
|
| 1084 |
+
},
|
| 1085 |
+
"backbone_ms": {
|
| 1086 |
+
"mean": 79.76332195103168,
|
| 1087 |
+
"median": 79.51081171631813,
|
| 1088 |
+
"p95": 80.94685152173042,
|
| 1089 |
+
"min": 79.24429979175329,
|
| 1090 |
+
"max": 81.1643386259675
|
| 1091 |
+
},
|
| 1092 |
+
"hidden_error": {
|
| 1093 |
+
"max_abs": 15.8125,
|
| 1094 |
+
"relative_l2": 0.06573484092950821
|
| 1095 |
+
},
|
| 1096 |
+
"drift": {
|
| 1097 |
+
"logits_max_abs": 0.03125,
|
| 1098 |
+
"logits_relative_l2": 0.007555761286739542,
|
| 1099 |
+
"decisions_matched": 2,
|
| 1100 |
+
"decisions_total": 2,
|
| 1101 |
+
"decision_agreement": 1.0,
|
| 1102 |
+
"reference_margin_min": 4.2109375,
|
| 1103 |
+
"reference_margin_median": 4.4228515625,
|
| 1104 |
+
"flips": [],
|
| 1105 |
+
"decisions": [
|
| 1106 |
+
{
|
| 1107 |
+
"record": 0,
|
| 1108 |
+
"question": 0,
|
| 1109 |
+
"reference_choice": 0,
|
| 1110 |
+
"candidate_choice": 0,
|
| 1111 |
+
"matched": true,
|
| 1112 |
+
"reference_margin": 4.2109375,
|
| 1113 |
+
"logits_max_abs": 0.03125,
|
| 1114 |
+
"margin_exceeds_twice_error": true
|
| 1115 |
+
},
|
| 1116 |
+
{
|
| 1117 |
+
"record": 0,
|
| 1118 |
+
"question": 1,
|
| 1119 |
+
"reference_choice": 1,
|
| 1120 |
+
"candidate_choice": 1,
|
| 1121 |
+
"matched": true,
|
| 1122 |
+
"reference_margin": 4.634765625,
|
| 1123 |
+
"logits_max_abs": 0.03125,
|
| 1124 |
+
"margin_exceeds_twice_error": true
|
| 1125 |
+
}
|
| 1126 |
+
]
|
| 1127 |
+
},
|
| 1128 |
+
"resident_vram_bytes": 30701897216,
|
| 1129 |
+
"peak_vram_bytes": 30759831040,
|
| 1130 |
+
"speedup_vs_fused_bf16": 0.796956165198466
|
| 1131 |
+
},
|
| 1132 |
+
{
|
| 1133 |
+
"case": 6,
|
| 1134 |
+
"tokens": 312,
|
| 1135 |
+
"total_ms": {
|
| 1136 |
+
"mean": 81.95691602304578,
|
| 1137 |
+
"median": 81.7097662948072,
|
| 1138 |
+
"p95": 82.6884787529707,
|
| 1139 |
+
"min": 81.4859289675951,
|
| 1140 |
+
"max": 87.34696544706821
|
| 1141 |
+
},
|
| 1142 |
+
"backbone_ms": {
|
| 1143 |
+
"mean": 79.5031982474029,
|
| 1144 |
+
"median": 78.90532445162535,
|
| 1145 |
+
"p95": 82.72027503699064,
|
| 1146 |
+
"min": 78.47033347934484,
|
| 1147 |
+
"max": 82.83683657646179
|
| 1148 |
+
},
|
| 1149 |
+
"hidden_error": {
|
| 1150 |
+
"max_abs": 11.75390625,
|
| 1151 |
+
"relative_l2": 0.09043972939252853
|
| 1152 |
+
},
|
| 1153 |
+
"drift": {
|
| 1154 |
+
"logits_max_abs": 0.0625,
|
| 1155 |
+
"logits_relative_l2": 0.01652767290549088,
|
| 1156 |
+
"decisions_matched": 2,
|
| 1157 |
+
"decisions_total": 2,
|
| 1158 |
+
"decision_agreement": 1.0,
|
| 1159 |
+
"reference_margin_min": 1.7109375,
|
| 1160 |
+
"reference_margin_median": 2.55859375,
|
| 1161 |
+
"flips": [],
|
| 1162 |
+
"decisions": [
|
| 1163 |
+
{
|
| 1164 |
+
"record": 0,
|
| 1165 |
+
"question": 0,
|
| 1166 |
+
"reference_choice": 2,
|
| 1167 |
+
"candidate_choice": 2,
|
| 1168 |
+
"matched": true,
|
| 1169 |
+
"reference_margin": 3.40625,
|
| 1170 |
+
"logits_max_abs": 0.0625,
|
| 1171 |
+
"margin_exceeds_twice_error": true
|
| 1172 |
+
},
|
| 1173 |
+
{
|
| 1174 |
+
"record": 0,
|
| 1175 |
+
"question": 1,
|
| 1176 |
+
"reference_choice": 0,
|
| 1177 |
+
"candidate_choice": 0,
|
| 1178 |
+
"matched": true,
|
| 1179 |
+
"reference_margin": 1.7109375,
|
| 1180 |
+
"logits_max_abs": 0.0625,
|
| 1181 |
+
"margin_exceeds_twice_error": true
|
| 1182 |
+
}
|
| 1183 |
+
]
|
| 1184 |
+
},
|
| 1185 |
+
"resident_vram_bytes": 30701897216,
|
| 1186 |
+
"peak_vram_bytes": 30762929664,
|
| 1187 |
+
"speedup_vs_fused_bf16": 0.8054773965934207
|
| 1188 |
+
}
|
| 1189 |
+
],
|
| 1190 |
+
"conversion": {
|
| 1191 |
+
"scope": "all",
|
| 1192 |
+
"linears": 400,
|
| 1193 |
+
"group_size": 256,
|
| 1194 |
+
"original_weight_bytes": 48653926400,
|
| 1195 |
+
"quantized_weight_bytes": 24350359552,
|
| 1196 |
+
"backend": "cute",
|
| 1197 |
+
"tile": [
|
| 1198 |
+
128,
|
| 1199 |
+
256
|
| 1200 |
+
],
|
| 1201 |
+
"cluster": [
|
| 1202 |
+
2,
|
| 1203 |
+
1
|
| 1204 |
+
],
|
| 1205 |
+
"calibration_index": "clef_text_int8_convrot_calibrated.safetensors.index.json",
|
| 1206 |
+
"total_quantized_linears": 400,
|
| 1207 |
+
"seconds": 3.488601409830153
|
| 1208 |
+
},
|
| 1209 |
+
"shared_input_exact": true,
|
| 1210 |
+
"patch_statistics": {
|
| 1211 |
+
"rms_calls": 43456,
|
| 1212 |
+
"swiglu_calls": 28672,
|
| 1213 |
+
"residual_calls": 28672,
|
| 1214 |
+
"delta_calls": 21504,
|
| 1215 |
+
"shared_quant_calls": 28672,
|
| 1216 |
+
"shared_attn_quant_calls": 21504,
|
| 1217 |
+
"fallbacks": 0
|
| 1218 |
+
}
|
| 1219 |
+
},
|
| 1220 |
+
"bf16_after": {
|
| 1221 |
+
"cases": [
|
| 1222 |
+
{
|
| 1223 |
+
"case": 0,
|
| 1224 |
+
"tokens": 1278,
|
| 1225 |
+
"total_ms": {
|
| 1226 |
+
"mean": 121.42789217953882,
|
| 1227 |
+
"median": 120.66404381766915,
|
| 1228 |
+
"p95": 127.8464999049902,
|
| 1229 |
+
"min": 116.93222634494305,
|
| 1230 |
+
"max": 133.81348084658384
|
| 1231 |
+
},
|
| 1232 |
+
"backbone_ms": {
|
| 1233 |
+
"mean": 117.61606192837158,
|
| 1234 |
+
"median": 117.35368939116597,
|
| 1235 |
+
"p95": 119.93260867893696,
|
| 1236 |
+
"min": 115.65397959202528,
|
| 1237 |
+
"max": 120.16370613127947
|
| 1238 |
+
},
|
| 1239 |
+
"hidden_error": {
|
| 1240 |
+
"max_abs": 0.0,
|
| 1241 |
+
"relative_l2": 0.0
|
| 1242 |
+
},
|
| 1243 |
+
"drift": {
|
| 1244 |
+
"logits_max_abs": 0.0,
|
| 1245 |
+
"logits_relative_l2": 0.0,
|
| 1246 |
+
"decisions_matched": 8,
|
| 1247 |
+
"decisions_total": 8,
|
| 1248 |
+
"decision_agreement": 1.0,
|
| 1249 |
+
"reference_margin_min": 0.0625,
|
| 1250 |
+
"reference_margin_median": 0.25,
|
| 1251 |
+
"flips": [],
|
| 1252 |
+
"decisions": [
|
| 1253 |
+
{
|
| 1254 |
+
"record": 0,
|
| 1255 |
+
"question": 0,
|
| 1256 |
+
"reference_choice": 0,
|
| 1257 |
+
"candidate_choice": 0,
|
| 1258 |
+
"matched": true,
|
| 1259 |
+
"reference_margin": 0.3671875,
|
| 1260 |
+
"logits_max_abs": 0.0,
|
| 1261 |
+
"margin_exceeds_twice_error": true
|
| 1262 |
+
},
|
| 1263 |
+
{
|
| 1264 |
+
"record": 0,
|
| 1265 |
+
"question": 1,
|
| 1266 |
+
"reference_choice": 0,
|
| 1267 |
+
"candidate_choice": 0,
|
| 1268 |
+
"matched": true,
|
| 1269 |
+
"reference_margin": 0.2265625,
|
| 1270 |
+
"logits_max_abs": 0.0,
|
| 1271 |
+
"margin_exceeds_twice_error": true
|
| 1272 |
+
},
|
| 1273 |
+
{
|
| 1274 |
+
"record": 0,
|
| 1275 |
+
"question": 2,
|
| 1276 |
+
"reference_choice": 0,
|
| 1277 |
+
"candidate_choice": 0,
|
| 1278 |
+
"matched": true,
|
| 1279 |
+
"reference_margin": 0.2265625,
|
| 1280 |
+
"logits_max_abs": 0.0,
|
| 1281 |
+
"margin_exceeds_twice_error": true
|
| 1282 |
+
},
|
| 1283 |
+
{
|
| 1284 |
+
"record": 0,
|
| 1285 |
+
"question": 3,
|
| 1286 |
+
"reference_choice": 0,
|
| 1287 |
+
"candidate_choice": 0,
|
| 1288 |
+
"matched": true,
|
| 1289 |
+
"reference_margin": 0.2734375,
|
| 1290 |
+
"logits_max_abs": 0.0,
|
| 1291 |
+
"margin_exceeds_twice_error": true
|
| 1292 |
+
},
|
| 1293 |
+
{
|
| 1294 |
+
"record": 0,
|
| 1295 |
+
"question": 4,
|
| 1296 |
+
"reference_choice": 0,
|
| 1297 |
+
"candidate_choice": 0,
|
| 1298 |
+
"matched": true,
|
| 1299 |
+
"reference_margin": 0.419921875,
|
| 1300 |
+
"logits_max_abs": 0.0,
|
| 1301 |
+
"margin_exceeds_twice_error": true
|
| 1302 |
+
},
|
| 1303 |
+
{
|
| 1304 |
+
"record": 0,
|
| 1305 |
+
"question": 5,
|
| 1306 |
+
"reference_choice": 0,
|
| 1307 |
+
"candidate_choice": 0,
|
| 1308 |
+
"matched": true,
|
| 1309 |
+
"reference_margin": 0.37109375,
|
| 1310 |
+
"logits_max_abs": 0.0,
|
| 1311 |
+
"margin_exceeds_twice_error": true
|
| 1312 |
+
},
|
| 1313 |
+
{
|
| 1314 |
+
"record": 0,
|
| 1315 |
+
"question": 6,
|
| 1316 |
+
"reference_choice": 0,
|
| 1317 |
+
"candidate_choice": 0,
|
| 1318 |
+
"matched": true,
|
| 1319 |
+
"reference_margin": 0.0625,
|
| 1320 |
+
"logits_max_abs": 0.0,
|
| 1321 |
+
"margin_exceeds_twice_error": true
|
| 1322 |
+
},
|
| 1323 |
+
{
|
| 1324 |
+
"record": 0,
|
| 1325 |
+
"question": 7,
|
| 1326 |
+
"reference_choice": 3,
|
| 1327 |
+
"candidate_choice": 3,
|
| 1328 |
+
"matched": true,
|
| 1329 |
+
"reference_margin": 0.1328125,
|
| 1330 |
+
"logits_max_abs": 0.0,
|
| 1331 |
+
"margin_exceeds_twice_error": true
|
| 1332 |
+
}
|
| 1333 |
+
]
|
| 1334 |
+
},
|
| 1335 |
+
"resident_vram_bytes": 55004415488,
|
| 1336 |
+
"peak_vram_bytes": 55225683456
|
| 1337 |
+
},
|
| 1338 |
+
{
|
| 1339 |
+
"case": 1,
|
| 1340 |
+
"tokens": 1502,
|
| 1341 |
+
"total_ms": {
|
| 1342 |
+
"mean": 140.32531641423702,
|
| 1343 |
+
"median": 140.29339561238885,
|
| 1344 |
+
"p95": 142.69945956766605,
|
| 1345 |
+
"min": 137.4212009832263,
|
| 1346 |
+
"max": 143.30243784934282
|
| 1347 |
+
},
|
| 1348 |
+
"backbone_ms": {
|
| 1349 |
+
"mean": 136.75211233397326,
|
| 1350 |
+
"median": 136.8663413450122,
|
| 1351 |
+
"p95": 138.3480206131935,
|
| 1352 |
+
"min": 132.71950464695692,
|
| 1353 |
+
"max": 139.0366218984127
|
| 1354 |
+
},
|
| 1355 |
+
"hidden_error": {
|
| 1356 |
+
"max_abs": 0.0,
|
| 1357 |
+
"relative_l2": 0.0
|
| 1358 |
+
},
|
| 1359 |
+
"drift": {
|
| 1360 |
+
"logits_max_abs": 0.0,
|
| 1361 |
+
"logits_relative_l2": 0.0,
|
| 1362 |
+
"decisions_matched": 8,
|
| 1363 |
+
"decisions_total": 8,
|
| 1364 |
+
"decision_agreement": 1.0,
|
| 1365 |
+
"reference_margin_min": 0.25,
|
| 1366 |
+
"reference_margin_median": 0.42578125,
|
| 1367 |
+
"flips": [],
|
| 1368 |
+
"decisions": [
|
| 1369 |
+
{
|
| 1370 |
+
"record": 0,
|
| 1371 |
+
"question": 0,
|
| 1372 |
+
"reference_choice": 0,
|
| 1373 |
+
"candidate_choice": 0,
|
| 1374 |
+
"matched": true,
|
| 1375 |
+
"reference_margin": 0.69921875,
|
| 1376 |
+
"logits_max_abs": 0.0,
|
| 1377 |
+
"margin_exceeds_twice_error": true
|
| 1378 |
+
},
|
| 1379 |
+
{
|
| 1380 |
+
"record": 0,
|
| 1381 |
+
"question": 1,
|
| 1382 |
+
"reference_choice": 0,
|
| 1383 |
+
"candidate_choice": 0,
|
| 1384 |
+
"matched": true,
|
| 1385 |
+
"reference_margin": 0.392578125,
|
| 1386 |
+
"logits_max_abs": 0.0,
|
| 1387 |
+
"margin_exceeds_twice_error": true
|
| 1388 |
+
},
|
| 1389 |
+
{
|
| 1390 |
+
"record": 0,
|
| 1391 |
+
"question": 2,
|
| 1392 |
+
"reference_choice": 0,
|
| 1393 |
+
"candidate_choice": 0,
|
| 1394 |
+
"matched": true,
|
| 1395 |
+
"reference_margin": 0.25,
|
| 1396 |
+
"logits_max_abs": 0.0,
|
| 1397 |
+
"margin_exceeds_twice_error": true
|
| 1398 |
+
},
|
| 1399 |
+
{
|
| 1400 |
+
"record": 0,
|
| 1401 |
+
"question": 3,
|
| 1402 |
+
"reference_choice": 0,
|
| 1403 |
+
"candidate_choice": 0,
|
| 1404 |
+
"matched": true,
|
| 1405 |
+
"reference_margin": 0.48828125,
|
| 1406 |
+
"logits_max_abs": 0.0,
|
| 1407 |
+
"margin_exceeds_twice_error": true
|
| 1408 |
+
},
|
| 1409 |
+
{
|
| 1410 |
+
"record": 0,
|
| 1411 |
+
"question": 4,
|
| 1412 |
+
"reference_choice": 0,
|
| 1413 |
+
"candidate_choice": 0,
|
| 1414 |
+
"matched": true,
|
| 1415 |
+
"reference_margin": 0.58984375,
|
| 1416 |
+
"logits_max_abs": 0.0,
|
| 1417 |
+
"margin_exceeds_twice_error": true
|
| 1418 |
+
},
|
| 1419 |
+
{
|
| 1420 |
+
"record": 0,
|
| 1421 |
+
"question": 5,
|
| 1422 |
+
"reference_choice": 0,
|
| 1423 |
+
"candidate_choice": 0,
|
| 1424 |
+
"matched": true,
|
| 1425 |
+
"reference_margin": 0.4296875,
|
| 1426 |
+
"logits_max_abs": 0.0,
|
| 1427 |
+
"margin_exceeds_twice_error": true
|
| 1428 |
+
},
|
| 1429 |
+
{
|
| 1430 |
+
"record": 0,
|
| 1431 |
+
"question": 6,
|
| 1432 |
+
"reference_choice": 0,
|
| 1433 |
+
"candidate_choice": 0,
|
| 1434 |
+
"matched": true,
|
| 1435 |
+
"reference_margin": 0.28515625,
|
| 1436 |
+
"logits_max_abs": 0.0,
|
| 1437 |
+
"margin_exceeds_twice_error": true
|
| 1438 |
+
},
|
| 1439 |
+
{
|
| 1440 |
+
"record": 0,
|
| 1441 |
+
"question": 7,
|
| 1442 |
+
"reference_choice": 0,
|
| 1443 |
+
"candidate_choice": 0,
|
| 1444 |
+
"matched": true,
|
| 1445 |
+
"reference_margin": 0.421875,
|
| 1446 |
+
"logits_max_abs": 0.0,
|
| 1447 |
+
"margin_exceeds_twice_error": true
|
| 1448 |
+
}
|
| 1449 |
+
]
|
| 1450 |
+
},
|
| 1451 |
+
"resident_vram_bytes": 55004415488,
|
| 1452 |
+
"peak_vram_bytes": 55265243648
|
| 1453 |
+
},
|
| 1454 |
+
{
|
| 1455 |
+
"case": 2,
|
| 1456 |
+
"tokens": 2270,
|
| 1457 |
+
"total_ms": {
|
| 1458 |
+
"mean": 207.1281478740275,
|
| 1459 |
+
"median": 206.86872629448771,
|
| 1460 |
+
"p95": 211.94536611437798,
|
| 1461 |
+
"min": 203.42395268380642,
|
| 1462 |
+
"max": 216.9617060571909
|
| 1463 |
+
},
|
| 1464 |
+
"backbone_ms": {
|
| 1465 |
+
"mean": 202.96321356048185,
|
| 1466 |
+
"median": 203.08450981974602,
|
| 1467 |
+
"p95": 204.29294928908348,
|
| 1468 |
+
"min": 200.65308827906847,
|
| 1469 |
+
"max": 204.85719107091427
|
| 1470 |
+
},
|
| 1471 |
+
"hidden_error": {
|
| 1472 |
+
"max_abs": 0.0,
|
| 1473 |
+
"relative_l2": 0.0
|
| 1474 |
+
},
|
| 1475 |
+
"drift": {
|
| 1476 |
+
"logits_max_abs": 0.0,
|
| 1477 |
+
"logits_relative_l2": 0.0,
|
| 1478 |
+
"decisions_matched": 8,
|
| 1479 |
+
"decisions_total": 8,
|
| 1480 |
+
"decision_agreement": 1.0,
|
| 1481 |
+
"reference_margin_min": 0.01953125,
|
| 1482 |
+
"reference_margin_median": 0.12109375,
|
| 1483 |
+
"flips": [],
|
| 1484 |
+
"decisions": [
|
| 1485 |
+
{
|
| 1486 |
+
"record": 0,
|
| 1487 |
+
"question": 0,
|
| 1488 |
+
"reference_choice": 0,
|
| 1489 |
+
"candidate_choice": 0,
|
| 1490 |
+
"matched": true,
|
| 1491 |
+
"reference_margin": 0.4765625,
|
| 1492 |
+
"logits_max_abs": 0.0,
|
| 1493 |
+
"margin_exceeds_twice_error": true
|
| 1494 |
+
},
|
| 1495 |
+
{
|
| 1496 |
+
"record": 0,
|
| 1497 |
+
"question": 1,
|
| 1498 |
+
"reference_choice": 0,
|
| 1499 |
+
"candidate_choice": 0,
|
| 1500 |
+
"matched": true,
|
| 1501 |
+
"reference_margin": 0.306640625,
|
| 1502 |
+
"logits_max_abs": 0.0,
|
| 1503 |
+
"margin_exceeds_twice_error": true
|
| 1504 |
+
},
|
| 1505 |
+
{
|
| 1506 |
+
"record": 0,
|
| 1507 |
+
"question": 2,
|
| 1508 |
+
"reference_choice": 0,
|
| 1509 |
+
"candidate_choice": 0,
|
| 1510 |
+
"matched": true,
|
| 1511 |
+
"reference_margin": 0.08203125,
|
| 1512 |
+
"logits_max_abs": 0.0,
|
| 1513 |
+
"margin_exceeds_twice_error": true
|
| 1514 |
+
},
|
| 1515 |
+
{
|
| 1516 |
+
"record": 0,
|
| 1517 |
+
"question": 3,
|
| 1518 |
+
"reference_choice": 0,
|
| 1519 |
+
"candidate_choice": 0,
|
| 1520 |
+
"matched": true,
|
| 1521 |
+
"reference_margin": 0.1171875,
|
| 1522 |
+
"logits_max_abs": 0.0,
|
| 1523 |
+
"margin_exceeds_twice_error": true
|
| 1524 |
+
},
|
| 1525 |
+
{
|
| 1526 |
+
"record": 0,
|
| 1527 |
+
"question": 4,
|
| 1528 |
+
"reference_choice": 0,
|
| 1529 |
+
"candidate_choice": 0,
|
| 1530 |
+
"matched": true,
|
| 1531 |
+
"reference_margin": 0.01953125,
|
| 1532 |
+
"logits_max_abs": 0.0,
|
| 1533 |
+
"margin_exceeds_twice_error": true
|
| 1534 |
+
},
|
| 1535 |
+
{
|
| 1536 |
+
"record": 0,
|
| 1537 |
+
"question": 5,
|
| 1538 |
+
"reference_choice": 0,
|
| 1539 |
+
"candidate_choice": 0,
|
| 1540 |
+
"matched": true,
|
| 1541 |
+
"reference_margin": 0.05859375,
|
| 1542 |
+
"logits_max_abs": 0.0,
|
| 1543 |
+
"margin_exceeds_twice_error": true
|
| 1544 |
+
},
|
| 1545 |
+
{
|
| 1546 |
+
"record": 0,
|
| 1547 |
+
"question": 6,
|
| 1548 |
+
"reference_choice": 3,
|
| 1549 |
+
"candidate_choice": 3,
|
| 1550 |
+
"matched": true,
|
| 1551 |
+
"reference_margin": 0.125,
|
| 1552 |
+
"logits_max_abs": 0.0,
|
| 1553 |
+
"margin_exceeds_twice_error": true
|
| 1554 |
+
},
|
| 1555 |
+
{
|
| 1556 |
+
"record": 0,
|
| 1557 |
+
"question": 7,
|
| 1558 |
+
"reference_choice": 0,
|
| 1559 |
+
"candidate_choice": 0,
|
| 1560 |
+
"matched": true,
|
| 1561 |
+
"reference_margin": 0.265625,
|
| 1562 |
+
"logits_max_abs": 0.0,
|
| 1563 |
+
"margin_exceeds_twice_error": true
|
| 1564 |
+
}
|
| 1565 |
+
]
|
| 1566 |
+
},
|
| 1567 |
+
"resident_vram_bytes": 55004415488,
|
| 1568 |
+
"peak_vram_bytes": 55398181376
|
| 1569 |
+
},
|
| 1570 |
+
{
|
| 1571 |
+
"case": 3,
|
| 1572 |
+
"tokens": 298,
|
| 1573 |
+
"total_ms": {
|
| 1574 |
+
"mean": 64.96169349799554,
|
| 1575 |
+
"median": 64.82085539028049,
|
| 1576 |
+
"p95": 65.15412591397762,
|
| 1577 |
+
"min": 64.52513858675957,
|
| 1578 |
+
"max": 68.82472150027752
|
| 1579 |
+
},
|
| 1580 |
+
"backbone_ms": {
|
| 1581 |
+
"mean": 61.811730451881886,
|
| 1582 |
+
"median": 61.80641520768404,
|
| 1583 |
+
"p95": 62.05604784190655,
|
| 1584 |
+
"min": 61.49221956729889,
|
| 1585 |
+
"max": 62.236362136900425
|
| 1586 |
+
},
|
| 1587 |
+
"hidden_error": {
|
| 1588 |
+
"max_abs": 0.0,
|
| 1589 |
+
"relative_l2": 0.0
|
| 1590 |
+
},
|
| 1591 |
+
"drift": {
|
| 1592 |
+
"logits_max_abs": 0.0,
|
| 1593 |
+
"logits_relative_l2": 0.0,
|
| 1594 |
+
"decisions_matched": 2,
|
| 1595 |
+
"decisions_total": 2,
|
| 1596 |
+
"decision_agreement": 1.0,
|
| 1597 |
+
"reference_margin_min": 4.31640625,
|
| 1598 |
+
"reference_margin_median": 4.353515625,
|
| 1599 |
+
"flips": [],
|
| 1600 |
+
"decisions": [
|
| 1601 |
+
{
|
| 1602 |
+
"record": 0,
|
| 1603 |
+
"question": 0,
|
| 1604 |
+
"reference_choice": 3,
|
| 1605 |
+
"candidate_choice": 3,
|
| 1606 |
+
"matched": true,
|
| 1607 |
+
"reference_margin": 4.390625,
|
| 1608 |
+
"logits_max_abs": 0.0,
|
| 1609 |
+
"margin_exceeds_twice_error": true
|
| 1610 |
+
},
|
| 1611 |
+
{
|
| 1612 |
+
"record": 0,
|
| 1613 |
+
"question": 1,
|
| 1614 |
+
"reference_choice": 1,
|
| 1615 |
+
"candidate_choice": 1,
|
| 1616 |
+
"matched": true,
|
| 1617 |
+
"reference_margin": 4.31640625,
|
| 1618 |
+
"logits_max_abs": 0.0,
|
| 1619 |
+
"margin_exceeds_twice_error": true
|
| 1620 |
+
}
|
| 1621 |
+
]
|
| 1622 |
+
},
|
| 1623 |
+
"resident_vram_bytes": 55004415488,
|
| 1624 |
+
"peak_vram_bytes": 55056573440
|
| 1625 |
+
},
|
| 1626 |
+
{
|
| 1627 |
+
"case": 4,
|
| 1628 |
+
"tokens": 303,
|
| 1629 |
+
"total_ms": {
|
| 1630 |
+
"mean": 64.5604116221269,
|
| 1631 |
+
"median": 64.56589605659246,
|
| 1632 |
+
"p95": 64.78502973914146,
|
| 1633 |
+
"min": 64.3211891874671,
|
| 1634 |
+
"max": 64.87731356173754
|
| 1635 |
+
},
|
| 1636 |
+
"backbone_ms": {
|
| 1637 |
+
"mean": 61.7412888755401,
|
| 1638 |
+
"median": 61.68849114328623,
|
| 1639 |
+
"p95": 62.02105898410082,
|
| 1640 |
+
"min": 61.47781666368246,
|
| 1641 |
+
"max": 62.256986275315285
|
| 1642 |
+
},
|
| 1643 |
+
"hidden_error": {
|
| 1644 |
+
"max_abs": 0.0,
|
| 1645 |
+
"relative_l2": 0.0
|
| 1646 |
+
},
|
| 1647 |
+
"drift": {
|
| 1648 |
+
"logits_max_abs": 0.0,
|
| 1649 |
+
"logits_relative_l2": 0.0,
|
| 1650 |
+
"decisions_matched": 2,
|
| 1651 |
+
"decisions_total": 2,
|
| 1652 |
+
"decision_agreement": 1.0,
|
| 1653 |
+
"reference_margin_min": 2.8984375,
|
| 1654 |
+
"reference_margin_median": 3.73046875,
|
| 1655 |
+
"flips": [],
|
| 1656 |
+
"decisions": [
|
| 1657 |
+
{
|
| 1658 |
+
"record": 0,
|
| 1659 |
+
"question": 0,
|
| 1660 |
+
"reference_choice": 2,
|
| 1661 |
+
"candidate_choice": 2,
|
| 1662 |
+
"matched": true,
|
| 1663 |
+
"reference_margin": 4.5625,
|
| 1664 |
+
"logits_max_abs": 0.0,
|
| 1665 |
+
"margin_exceeds_twice_error": true
|
| 1666 |
+
},
|
| 1667 |
+
{
|
| 1668 |
+
"record": 0,
|
| 1669 |
+
"question": 1,
|
| 1670 |
+
"reference_choice": 0,
|
| 1671 |
+
"candidate_choice": 0,
|
| 1672 |
+
"matched": true,
|
| 1673 |
+
"reference_margin": 2.8984375,
|
| 1674 |
+
"logits_max_abs": 0.0,
|
| 1675 |
+
"margin_exceeds_twice_error": true
|
| 1676 |
+
}
|
| 1677 |
+
]
|
| 1678 |
+
},
|
| 1679 |
+
"resident_vram_bytes": 55004415488,
|
| 1680 |
+
"peak_vram_bytes": 55058310656
|
| 1681 |
+
},
|
| 1682 |
+
{
|
| 1683 |
+
"case": 5,
|
| 1684 |
+
"tokens": 304,
|
| 1685 |
+
"total_ms": {
|
| 1686 |
+
"mean": 64.69714563960831,
|
| 1687 |
+
"median": 64.54729475080967,
|
| 1688 |
+
"p95": 64.86918963491917,
|
| 1689 |
+
"min": 64.34616167098284,
|
| 1690 |
+
"max": 68.42743512243032
|
| 1691 |
+
},
|
| 1692 |
+
"backbone_ms": {
|
| 1693 |
+
"mean": 61.82925890510281,
|
| 1694 |
+
"median": 61.83003494516015,
|
| 1695 |
+
"p95": 62.04524822533131,
|
| 1696 |
+
"min": 61.482759192585945,
|
| 1697 |
+
"max": 62.447430565953255
|
| 1698 |
+
},
|
| 1699 |
+
"hidden_error": {
|
| 1700 |
+
"max_abs": 0.0,
|
| 1701 |
+
"relative_l2": 0.0
|
| 1702 |
+
},
|
| 1703 |
+
"drift": {
|
| 1704 |
+
"logits_max_abs": 0.0,
|
| 1705 |
+
"logits_relative_l2": 0.0,
|
| 1706 |
+
"decisions_matched": 2,
|
| 1707 |
+
"decisions_total": 2,
|
| 1708 |
+
"decision_agreement": 1.0,
|
| 1709 |
+
"reference_margin_min": 4.2109375,
|
| 1710 |
+
"reference_margin_median": 4.4228515625,
|
| 1711 |
+
"flips": [],
|
| 1712 |
+
"decisions": [
|
| 1713 |
+
{
|
| 1714 |
+
"record": 0,
|
| 1715 |
+
"question": 0,
|
| 1716 |
+
"reference_choice": 0,
|
| 1717 |
+
"candidate_choice": 0,
|
| 1718 |
+
"matched": true,
|
| 1719 |
+
"reference_margin": 4.2109375,
|
| 1720 |
+
"logits_max_abs": 0.0,
|
| 1721 |
+
"margin_exceeds_twice_error": true
|
| 1722 |
+
},
|
| 1723 |
+
{
|
| 1724 |
+
"record": 0,
|
| 1725 |
+
"question": 1,
|
| 1726 |
+
"reference_choice": 1,
|
| 1727 |
+
"candidate_choice": 1,
|
| 1728 |
+
"matched": true,
|
| 1729 |
+
"reference_margin": 4.634765625,
|
| 1730 |
+
"logits_max_abs": 0.0,
|
| 1731 |
+
"margin_exceeds_twice_error": true
|
| 1732 |
+
}
|
| 1733 |
+
]
|
| 1734 |
+
},
|
| 1735 |
+
"resident_vram_bytes": 55004415488,
|
| 1736 |
+
"peak_vram_bytes": 55058413056
|
| 1737 |
+
},
|
| 1738 |
+
{
|
| 1739 |
+
"case": 6,
|
| 1740 |
+
"tokens": 312,
|
| 1741 |
+
"total_ms": {
|
| 1742 |
+
"mean": 65.49102198332548,
|
| 1743 |
+
"median": 65.15130214393139,
|
| 1744 |
+
"p95": 67.28412676602602,
|
| 1745 |
+
"min": 64.53924346715212,
|
| 1746 |
+
"max": 69.64690797030926
|
| 1747 |
+
},
|
| 1748 |
+
"backbone_ms": {
|
| 1749 |
+
"mean": 62.528794786582395,
|
| 1750 |
+
"median": 62.569474801421165,
|
| 1751 |
+
"p95": 63.048550859093666,
|
| 1752 |
+
"min": 62.03409377485514,
|
| 1753 |
+
"max": 63.295284286141396
|
| 1754 |
+
},
|
| 1755 |
+
"hidden_error": {
|
| 1756 |
+
"max_abs": 0.0,
|
| 1757 |
+
"relative_l2": 0.0
|
| 1758 |
+
},
|
| 1759 |
+
"drift": {
|
| 1760 |
+
"logits_max_abs": 0.0,
|
| 1761 |
+
"logits_relative_l2": 0.0,
|
| 1762 |
+
"decisions_matched": 2,
|
| 1763 |
+
"decisions_total": 2,
|
| 1764 |
+
"decision_agreement": 1.0,
|
| 1765 |
+
"reference_margin_min": 1.7109375,
|
| 1766 |
+
"reference_margin_median": 2.55859375,
|
| 1767 |
+
"flips": [],
|
| 1768 |
+
"decisions": [
|
| 1769 |
+
{
|
| 1770 |
+
"record": 0,
|
| 1771 |
+
"question": 0,
|
| 1772 |
+
"reference_choice": 2,
|
| 1773 |
+
"candidate_choice": 2,
|
| 1774 |
+
"matched": true,
|
| 1775 |
+
"reference_margin": 3.40625,
|
| 1776 |
+
"logits_max_abs": 0.0,
|
| 1777 |
+
"margin_exceeds_twice_error": true
|
| 1778 |
+
},
|
| 1779 |
+
{
|
| 1780 |
+
"record": 0,
|
| 1781 |
+
"question": 1,
|
| 1782 |
+
"reference_choice": 0,
|
| 1783 |
+
"candidate_choice": 0,
|
| 1784 |
+
"matched": true,
|
| 1785 |
+
"reference_margin": 1.7109375,
|
| 1786 |
+
"logits_max_abs": 0.0,
|
| 1787 |
+
"margin_exceeds_twice_error": true
|
| 1788 |
+
}
|
| 1789 |
+
]
|
| 1790 |
+
},
|
| 1791 |
+
"resident_vram_bytes": 55004415488,
|
| 1792 |
+
"peak_vram_bytes": 55060240896
|
| 1793 |
+
}
|
| 1794 |
+
],
|
| 1795 |
+
"conversion": null,
|
| 1796 |
+
"shared_input_exact": null,
|
| 1797 |
+
"patch_statistics": {
|
| 1798 |
+
"rms_calls": 43456,
|
| 1799 |
+
"swiglu_calls": 28672,
|
| 1800 |
+
"residual_calls": 28672,
|
| 1801 |
+
"delta_calls": 21504,
|
| 1802 |
+
"shared_quant_calls": 0,
|
| 1803 |
+
"shared_attn_quant_calls": 0,
|
| 1804 |
+
"fallbacks": 0
|
| 1805 |
+
}
|
| 1806 |
+
}
|
| 1807 |
+
},
|
| 1808 |
+
"quality_scope": "BF16 agreement only, not task accuracy or production validation",
|
| 1809 |
+
"records_source": "synthetic",
|
| 1810 |
+
"bracketed": true
|
| 1811 |
+
}
|
results/calibration.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
runtime/clef-bench.py
ADDED
|
@@ -0,0 +1,479 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Profile Cloudflare Clef backbone and joint-head latency."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import importlib.util
|
| 8 |
+
import json
|
| 9 |
+
import os
|
| 10 |
+
import statistics
|
| 11 |
+
import sys
|
| 12 |
+
import time
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
from typing import Any
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _prepend_env_path(name: str, value: Path) -> None:
|
| 18 |
+
if not value.exists():
|
| 19 |
+
return
|
| 20 |
+
current = os.environ.get(name)
|
| 21 |
+
value_text = str(value)
|
| 22 |
+
if not current:
|
| 23 |
+
os.environ[name] = value_text
|
| 24 |
+
return
|
| 25 |
+
parts = current.split(os.pathsep)
|
| 26 |
+
if value_text not in parts:
|
| 27 |
+
os.environ[name] = os.pathsep.join([value_text, current])
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _configure_cuda_compiler_env() -> None:
|
| 31 |
+
cuda_home = os.environ.get("CUDA_HOME")
|
| 32 |
+
candidates = [Path(cuda_home)] if cuda_home else []
|
| 33 |
+
candidates.extend([Path("/usr/local/cuda"), Path("/usr/local/cuda-13.0"), Path("/usr/local/cuda-12.8")])
|
| 34 |
+
cuda_root = next((path.resolve() for path in candidates if path.exists()), None)
|
| 35 |
+
if cuda_root is None:
|
| 36 |
+
return
|
| 37 |
+
|
| 38 |
+
os.environ.setdefault("CUDA_HOME", str(cuda_root))
|
| 39 |
+
ptxas = cuda_root / "bin" / "ptxas"
|
| 40 |
+
if ptxas.exists():
|
| 41 |
+
os.environ.setdefault("TRITON_PTXAS_PATH", str(ptxas))
|
| 42 |
+
_prepend_env_path("PATH", ptxas.parent)
|
| 43 |
+
|
| 44 |
+
target = cuda_root / "targets" / "x86_64-linux"
|
| 45 |
+
_prepend_env_path("CPATH", target / "include")
|
| 46 |
+
_prepend_env_path("LIBRARY_PATH", target / "lib" / "stubs")
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
_configure_cuda_compiler_env()
|
| 50 |
+
|
| 51 |
+
import torch
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
DEFAULT_MODEL = "Cloudflare/clef"
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def parse_args() -> argparse.Namespace:
|
| 58 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 59 |
+
parser.add_argument("--model", default=DEFAULT_MODEL)
|
| 60 |
+
parser.add_argument("--local-files-only", action="store_true")
|
| 61 |
+
parser.add_argument("--device", default="cuda")
|
| 62 |
+
parser.add_argument("--dtype", choices=["bfloat16", "float16", "float32"], default="bfloat16")
|
| 63 |
+
parser.add_argument("--records", type=int, default=1)
|
| 64 |
+
parser.add_argument("--questions", type=int, default=8)
|
| 65 |
+
parser.add_argument("--choices", type=int, default=4)
|
| 66 |
+
parser.add_argument("--state-words", type=int, default=256)
|
| 67 |
+
parser.add_argument("--max-length", type=int, default=16384)
|
| 68 |
+
parser.add_argument("--warmup", type=int, default=2)
|
| 69 |
+
parser.add_argument("--repeat", type=int, default=10)
|
| 70 |
+
parser.add_argument("--profile", action="store_true")
|
| 71 |
+
parser.add_argument("--profile-stages", action="store_true")
|
| 72 |
+
parser.add_argument("--capture-nsys", action="store_true")
|
| 73 |
+
parser.add_argument("--profile-iterations", type=int, default=3)
|
| 74 |
+
parser.add_argument("--image-size", type=int, default=0)
|
| 75 |
+
parser.add_argument("--image-count", type=int, default=1)
|
| 76 |
+
parser.add_argument("--vision-backend", choices=["reference", "no-sync", "cute", "cute-fused", "cute-graph"], default="reference")
|
| 77 |
+
parser.add_argument("--compare-vision", action="store_true")
|
| 78 |
+
parser.add_argument("--text-backend", choices=["reference", "cute-rms", "cute-swiglu", "cute", "cute-gva", "cute-gates", "cute-all", "cute-heads"], default="reference")
|
| 79 |
+
parser.add_argument("--compare-text", action="store_true")
|
| 80 |
+
parser.add_argument("--compile-head", action="store_true")
|
| 81 |
+
parser.add_argument(
|
| 82 |
+
"--linear-attn-backend",
|
| 83 |
+
choices=["auto", "torch"],
|
| 84 |
+
default="auto",
|
| 85 |
+
help="Use Transformers' selected Qwen linear-attention kernels, or force the torch fallback.",
|
| 86 |
+
)
|
| 87 |
+
parser.add_argument("--output", type=Path, default=Path("clef-bench.json"))
|
| 88 |
+
return parser.parse_args()
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def dtype_from_name(name: str) -> torch.dtype:
|
| 92 |
+
return {
|
| 93 |
+
"bfloat16": torch.bfloat16,
|
| 94 |
+
"float16": torch.float16,
|
| 95 |
+
"float32": torch.float32,
|
| 96 |
+
}[name]
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def load_joint_schema_module(model_path: Path):
|
| 100 |
+
module_path = model_path / "joint_schema_model.py"
|
| 101 |
+
spec = importlib.util.spec_from_file_location("clef_joint_schema_model", module_path)
|
| 102 |
+
if spec is None or spec.loader is None:
|
| 103 |
+
raise RuntimeError(f"Could not import {module_path}")
|
| 104 |
+
module = importlib.util.module_from_spec(spec)
|
| 105 |
+
sys.modules[spec.name] = module
|
| 106 |
+
spec.loader.exec_module(module)
|
| 107 |
+
return module
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def make_record(index: int, questions: int, choices: int, state_words: int) -> dict[str, Any]:
|
| 111 |
+
words = [
|
| 112 |
+
"invoice",
|
| 113 |
+
"router",
|
| 114 |
+
"customer",
|
| 115 |
+
"risk",
|
| 116 |
+
"policy",
|
| 117 |
+
"payment",
|
| 118 |
+
"service",
|
| 119 |
+
"contract",
|
| 120 |
+
"evidence",
|
| 121 |
+
"priority",
|
| 122 |
+
]
|
| 123 |
+
state = {
|
| 124 |
+
"case_id": f"bench-{index}",
|
| 125 |
+
"summary": " ".join(words[i % len(words)] for i in range(state_words)),
|
| 126 |
+
"amount": 1250 + index,
|
| 127 |
+
"currency": "USD",
|
| 128 |
+
"signals": {
|
| 129 |
+
"escalated": index % 2 == 0,
|
| 130 |
+
"region": "us",
|
| 131 |
+
"latency_ms": 217,
|
| 132 |
+
},
|
| 133 |
+
}
|
| 134 |
+
schema = {}
|
| 135 |
+
for q_idx in range(questions):
|
| 136 |
+
schema[f"field_{q_idx:03d}"] = {
|
| 137 |
+
"type": "choice",
|
| 138 |
+
"instructions": f"Choose the best route for field {q_idx}.",
|
| 139 |
+
"criteria": {
|
| 140 |
+
f"option_{choice_idx:02d}": f"Evidence pattern {choice_idx} for field {q_idx}."
|
| 141 |
+
for choice_idx in range(choices)
|
| 142 |
+
},
|
| 143 |
+
}
|
| 144 |
+
return {"id": f"bench-{index}", "state": state, "questions": schema}
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def sync(device: torch.device) -> None:
|
| 148 |
+
if device.type == "cuda":
|
| 149 |
+
torch.cuda.synchronize(device)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def timed(fn, device: torch.device) -> tuple[Any, float]:
|
| 153 |
+
sync(device)
|
| 154 |
+
start = time.perf_counter()
|
| 155 |
+
value = fn()
|
| 156 |
+
sync(device)
|
| 157 |
+
return value, time.perf_counter() - start
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def percentile(values: list[float], pct: float) -> float:
|
| 161 |
+
if not values:
|
| 162 |
+
return float("nan")
|
| 163 |
+
ordered = sorted(values)
|
| 164 |
+
index = min(len(ordered) - 1, max(0, round((len(ordered) - 1) * pct)))
|
| 165 |
+
return ordered[index]
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def summarize_ms(values: list[float]) -> dict[str, float]:
|
| 169 |
+
return {
|
| 170 |
+
"mean": statistics.fmean(values) * 1000,
|
| 171 |
+
"median": statistics.median(values) * 1000,
|
| 172 |
+
"p95": percentile(values, 0.95) * 1000,
|
| 173 |
+
"min": min(values) * 1000,
|
| 174 |
+
"max": max(values) * 1000,
|
| 175 |
+
}
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def force_torch_linear_attention(text_model: torch.nn.Module) -> int:
|
| 179 |
+
module = importlib.import_module(text_model.__class__.__module__)
|
| 180 |
+
rms_norm_gated_cls = getattr(
|
| 181 |
+
module,
|
| 182 |
+
"Qwen3_5RMSNormGated",
|
| 183 |
+
getattr(module, "Qwen3NextRMSNormGated", None),
|
| 184 |
+
)
|
| 185 |
+
if rms_norm_gated_cls is None:
|
| 186 |
+
raise RuntimeError("Could not find the Qwen gated RMSNorm fallback class")
|
| 187 |
+
replaced = 0
|
| 188 |
+
for layer in getattr(text_model, "layers", []):
|
| 189 |
+
linear_attn = getattr(layer, "linear_attn", None)
|
| 190 |
+
if linear_attn is None:
|
| 191 |
+
continue
|
| 192 |
+
linear_attn.causal_conv1d_fn = None
|
| 193 |
+
linear_attn.causal_conv1d_update = module.torch_causal_conv1d_update
|
| 194 |
+
linear_attn.chunk_gated_delta_rule = module.torch_chunk_gated_delta_rule
|
| 195 |
+
linear_attn.recurrent_gated_delta_rule = module.torch_recurrent_gated_delta_rule
|
| 196 |
+
if not isinstance(linear_attn.norm, rms_norm_gated_cls):
|
| 197 |
+
original_norm = linear_attn.norm
|
| 198 |
+
replacement = rms_norm_gated_cls(
|
| 199 |
+
linear_attn.head_v_dim,
|
| 200 |
+
eps=linear_attn.layer_norm_epsilon,
|
| 201 |
+
).to(device=original_norm.weight.device, dtype=original_norm.weight.dtype)
|
| 202 |
+
replacement.weight.data.copy_(original_norm.weight.data)
|
| 203 |
+
linear_attn.norm = replacement
|
| 204 |
+
replaced += 1
|
| 205 |
+
if replaced == 0:
|
| 206 |
+
raise RuntimeError("No Qwen linear-attention layers were found to patch")
|
| 207 |
+
return replaced
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def main() -> None:
|
| 211 |
+
args = parse_args()
|
| 212 |
+
if args.compare_text and (args.text_backend == "reference" or args.compare_vision):
|
| 213 |
+
raise ValueError("--compare-text requires a text candidate and cannot be combined with --compare-vision")
|
| 214 |
+
if min(args.repeat, args.warmup, args.profile_iterations) < 1:
|
| 215 |
+
raise ValueError("Repeat, warmup, and profile iterations must be positive")
|
| 216 |
+
if args.linear_attn_backend == "torch" and args.text_backend in {"cute-gva", "cute-all", "cute-heads"}:
|
| 217 |
+
raise ValueError("Native GVA requires the FLA backend, not --linear-attn-backend torch")
|
| 218 |
+
os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
|
| 219 |
+
device = torch.device(args.device)
|
| 220 |
+
dtype = dtype_from_name(args.dtype)
|
| 221 |
+
|
| 222 |
+
from huggingface_hub import snapshot_download
|
| 223 |
+
from transformers.utils.import_utils import (
|
| 224 |
+
is_causal_conv1d_available,
|
| 225 |
+
is_flash_linear_attention_available,
|
| 226 |
+
)
|
| 227 |
+
|
| 228 |
+
model_path = Path(args.model)
|
| 229 |
+
if not model_path.is_dir():
|
| 230 |
+
model_path = Path(snapshot_download(args.model, local_files_only=args.local_files_only))
|
| 231 |
+
joint_schema = load_joint_schema_module(model_path)
|
| 232 |
+
model, processor = joint_schema.load_release_model(model_path, device=device, dtype=dtype)
|
| 233 |
+
if args.compile_head:
|
| 234 |
+
model.head = torch.compile(model.head, mode="reduce-overhead")
|
| 235 |
+
|
| 236 |
+
records = [
|
| 237 |
+
make_record(index, args.questions, args.choices, args.state_words)
|
| 238 |
+
for index in range(args.records)
|
| 239 |
+
]
|
| 240 |
+
if args.image_size:
|
| 241 |
+
from PIL import Image
|
| 242 |
+
import numpy as np
|
| 243 |
+
|
| 244 |
+
for i, record in enumerate(records):
|
| 245 |
+
rng = np.random.default_rng(1234 + i)
|
| 246 |
+
record["images"] = [
|
| 247 |
+
Image.fromarray(rng.integers(0, 256, (args.image_size, args.image_size, 3), dtype=np.uint8))
|
| 248 |
+
for _ in range(args.image_count)
|
| 249 |
+
]
|
| 250 |
+
elif args.vision_backend != "reference" or args.compare_vision:
|
| 251 |
+
raise ValueError("Vision benchmarking requires --image-size")
|
| 252 |
+
if args.compare_vision and args.vision_backend == "reference":
|
| 253 |
+
raise ValueError("--compare-vision requires a candidate vision backend")
|
| 254 |
+
encoded, encode_seconds = timed(
|
| 255 |
+
lambda: [
|
| 256 |
+
joint_schema.encode_record(
|
| 257 |
+
processor.tokenizer,
|
| 258 |
+
record,
|
| 259 |
+
max_length=args.max_length,
|
| 260 |
+
processor=processor,
|
| 261 |
+
)
|
| 262 |
+
for record in records
|
| 263 |
+
],
|
| 264 |
+
device,
|
| 265 |
+
)
|
| 266 |
+
batch = joint_schema.collate_records(encoded, processor.tokenizer.pad_token_id, device)
|
| 267 |
+
|
| 268 |
+
base_model = (
|
| 269 |
+
model.language_model.get_base_model()
|
| 270 |
+
if hasattr(model.language_model, "get_base_model")
|
| 271 |
+
else model.language_model
|
| 272 |
+
)
|
| 273 |
+
text_model = base_model.model
|
| 274 |
+
media = batch.get("media") or {}
|
| 275 |
+
if not media and hasattr(text_model, "language_model"):
|
| 276 |
+
text_model = text_model.language_model
|
| 277 |
+
forced_linear_attn_layers = 0
|
| 278 |
+
if args.linear_attn_backend == "torch":
|
| 279 |
+
forced_linear_attn_layers = force_torch_linear_attention(getattr(text_model, "language_model", text_model))
|
| 280 |
+
|
| 281 |
+
def run_backbone():
|
| 282 |
+
return text_model(
|
| 283 |
+
input_ids=batch["input_ids"],
|
| 284 |
+
attention_mask=batch["attention_mask"],
|
| 285 |
+
use_cache=False,
|
| 286 |
+
return_dict=True,
|
| 287 |
+
**media,
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
def run_once() -> tuple[Any, Any]:
|
| 291 |
+
outputs = run_backbone()
|
| 292 |
+
logits = model.head(
|
| 293 |
+
outputs.last_hidden_state,
|
| 294 |
+
batch["input_ids"],
|
| 295 |
+
batch["attention_mask"],
|
| 296 |
+
batch["records"],
|
| 297 |
+
base_model.get_output_embeddings().weight,
|
| 298 |
+
)
|
| 299 |
+
return outputs, logits
|
| 300 |
+
|
| 301 |
+
vision_comparison = None
|
| 302 |
+
text_comparison = None
|
| 303 |
+
with torch.inference_mode():
|
| 304 |
+
if args.vision_backend != "reference":
|
| 305 |
+
from clef_vision_rope import patch_attention
|
| 306 |
+
|
| 307 |
+
if args.compare_vision:
|
| 308 |
+
for _ in range(args.warmup):
|
| 309 |
+
run_once()
|
| 310 |
+
reference_logits = [[v.clone() for v in r] for r in run_once()[1]]
|
| 311 |
+
reference_times = [timed(run_once, device)[1] for _ in range(args.repeat)]
|
| 312 |
+
grid = media["image_grid_thw"].cpu().tolist()
|
| 313 |
+
lengths = tuple(h * w for t, h, w in grid for _ in range(t))
|
| 314 |
+
def install_vision():
|
| 315 |
+
if args.vision_backend in {"cute-fused", "cute-graph"}:
|
| 316 |
+
from clef_vision_prepared import patch_prepared_vision
|
| 317 |
+
return patch_prepared_vision(base_model.model.visual, capture=args.vision_backend == "cute-graph")
|
| 318 |
+
return patch_attention(base_model.model.visual, rope=args.vision_backend == "cute", lengths=lengths)
|
| 319 |
+
|
| 320 |
+
restore_vision = install_vision()
|
| 321 |
+
if args.compare_vision:
|
| 322 |
+
actual_logits = run_once()[1]
|
| 323 |
+
expected = torch.cat([v.flatten() for r in reference_logits for v in r])
|
| 324 |
+
actual = torch.cat([v.flatten() for r in actual_logits for v in r])
|
| 325 |
+
delta = (actual.float() - expected.float())
|
| 326 |
+
matches = sum(int(a.argmax() == b.argmax()) for ar, br in zip(actual_logits, reference_logits)
|
| 327 |
+
for a, b in zip(ar, br))
|
| 328 |
+
count = sum(len(r) for r in reference_logits)
|
| 329 |
+
vision_comparison = {"reference_total_ms": summarize_ms(reference_times),
|
| 330 |
+
"logits_max_abs": delta.abs().max().item(),
|
| 331 |
+
"logits_relative_l2": (delta.norm() / expected.float().norm().clamp_min(1e-12)).item(),
|
| 332 |
+
"decisions_matched": matches, "decisions_total": count}
|
| 333 |
+
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
| 334 |
+
if matches != count:
|
| 335 |
+
raise AssertionError("CuTe vision changed a CLEF decision")
|
| 336 |
+
if args.text_backend != "reference":
|
| 337 |
+
from clef_text_cute import patch_text
|
| 338 |
+
if args.compare_text:
|
| 339 |
+
for _ in range(args.warmup):
|
| 340 |
+
run_once()
|
| 341 |
+
text_expected = torch.cat([v.flatten() for r in run_once()[1] for v in r]).clone()
|
| 342 |
+
text_reference_times = [timed(run_once, device)[1] for _ in range(args.repeat)]
|
| 343 |
+
def install_text():
|
| 344 |
+
return patch_text(getattr(text_model, "language_model", text_model),
|
| 345 |
+
rms=args.text_backend != "cute-swiglu",
|
| 346 |
+
activation=args.text_backend != "cute-rms",
|
| 347 |
+
gva=args.text_backend in {"cute-gva", "cute-all", "cute-heads"},
|
| 348 |
+
gates=args.text_backend in {"cute-gates", "cute-all", "cute-heads"},
|
| 349 |
+
heads=args.text_backend == "cute-heads")
|
| 350 |
+
restore_text = install_text()
|
| 351 |
+
if args.compare_text:
|
| 352 |
+
actual_logits = run_once()[1]
|
| 353 |
+
actual = torch.cat([v.flatten() for r in actual_logits for v in r])
|
| 354 |
+
torch.testing.assert_close(actual, text_expected, atol=0, rtol=0)
|
| 355 |
+
text_comparison = {"reference_total_ms": summarize_ms(text_reference_times),
|
| 356 |
+
"logits_max_abs": float((actual.float() - text_expected.float()).abs().max()),
|
| 357 |
+
"decisions_total": sum(len(r) for r in actual_logits), "exact": True}
|
| 358 |
+
for _ in range(args.warmup):
|
| 359 |
+
run_once()
|
| 360 |
+
sync(device)
|
| 361 |
+
|
| 362 |
+
backbone_times: list[float] = []
|
| 363 |
+
head_times: list[float] = []
|
| 364 |
+
total_times: list[float] = []
|
| 365 |
+
for _ in range(args.repeat):
|
| 366 |
+
sync(device)
|
| 367 |
+
total_start = time.perf_counter()
|
| 368 |
+
outputs, backbone_elapsed = timed(run_backbone, device)
|
| 369 |
+
_, head_elapsed = timed(
|
| 370 |
+
lambda: model.head(
|
| 371 |
+
outputs.last_hidden_state,
|
| 372 |
+
batch["input_ids"],
|
| 373 |
+
batch["attention_mask"],
|
| 374 |
+
batch["records"],
|
| 375 |
+
base_model.get_output_embeddings().weight,
|
| 376 |
+
),
|
| 377 |
+
device,
|
| 378 |
+
)
|
| 379 |
+
sync(device)
|
| 380 |
+
total_times.append(time.perf_counter() - total_start)
|
| 381 |
+
backbone_times.append(backbone_elapsed)
|
| 382 |
+
head_times.append(head_elapsed)
|
| 383 |
+
|
| 384 |
+
if args.compare_vision:
|
| 385 |
+
candidate_times = [timed(run_once, device)[1] for _ in range(args.repeat)]
|
| 386 |
+
if hasattr(restore_vision, "statistics"):
|
| 387 |
+
vision_comparison["prepared_cache"] = dict(restore_vision.statistics)
|
| 388 |
+
if not restore_vision.statistics["calls"] or restore_vision.statistics["fallbacks"]:
|
| 389 |
+
raise AssertionError("Prepared vision was not used for every candidate call")
|
| 390 |
+
restore_vision()
|
| 391 |
+
reference_after = [timed(run_once, device)[1] for _ in range(args.repeat)]
|
| 392 |
+
vision_comparison["candidate_total_ms"] = summarize_ms(candidate_times)
|
| 393 |
+
vision_comparison["reference_after_total_ms"] = summarize_ms(reference_after)
|
| 394 |
+
vision_comparison["speedup"] = ((statistics.median(reference_times) + statistics.median(reference_after))
|
| 395 |
+
/ (2 * statistics.median(candidate_times)))
|
| 396 |
+
install_vision()
|
| 397 |
+
|
| 398 |
+
if args.compare_text:
|
| 399 |
+
candidate_times = [timed(run_once, device)[1] for _ in range(args.repeat)]
|
| 400 |
+
text_comparison["patch_statistics"] = dict(restore_text.statistics)
|
| 401 |
+
restore_text()
|
| 402 |
+
reference_after = [timed(run_once, device)[1] for _ in range(args.repeat)]
|
| 403 |
+
text_comparison["candidate_total_ms"] = summarize_ms(candidate_times)
|
| 404 |
+
text_comparison["reference_after_total_ms"] = summarize_ms(reference_after)
|
| 405 |
+
text_comparison["speedup"] = ((statistics.median(text_reference_times) + statistics.median(reference_after))
|
| 406 |
+
/ (2 * statistics.median(candidate_times)))
|
| 407 |
+
restore_text = install_text()
|
| 408 |
+
|
| 409 |
+
profile_table = None
|
| 410 |
+
stage_profile = None
|
| 411 |
+
if args.profile:
|
| 412 |
+
activities = [torch.profiler.ProfilerActivity.CPU]
|
| 413 |
+
if device.type == "cuda":
|
| 414 |
+
activities.append(torch.profiler.ProfilerActivity.CUDA)
|
| 415 |
+
with torch.profiler.profile(
|
| 416 |
+
activities=activities,
|
| 417 |
+
record_shapes=True,
|
| 418 |
+
profile_memory=True,
|
| 419 |
+
with_stack=False,
|
| 420 |
+
) as prof:
|
| 421 |
+
run_once()
|
| 422 |
+
profile_table = prof.key_averages().table(
|
| 423 |
+
sort_by="cuda_time_total" if device.type == "cuda" else "cpu_time_total",
|
| 424 |
+
row_limit=40,
|
| 425 |
+
)
|
| 426 |
+
if args.profile_stages or args.capture_nsys:
|
| 427 |
+
from clef_text_profile import profile_text
|
| 428 |
+
stage_profile = profile_text(run_once, getattr(text_model, "language_model", text_model),
|
| 429 |
+
args.output.parent / (args.output.stem + "-profile"), head=model.head,
|
| 430 |
+
iterations=args.profile_iterations, capture_nsys=args.capture_nsys)
|
| 431 |
+
|
| 432 |
+
max_memory = None
|
| 433 |
+
if device.type == "cuda":
|
| 434 |
+
max_memory = torch.cuda.max_memory_allocated(device)
|
| 435 |
+
|
| 436 |
+
result = {
|
| 437 |
+
"model": args.model,
|
| 438 |
+
"model_path": str(model_path),
|
| 439 |
+
"device": str(device),
|
| 440 |
+
"device_name": torch.cuda.get_device_name(device) if device.type == "cuda" else None,
|
| 441 |
+
"dtype": args.dtype,
|
| 442 |
+
"torch": torch.__version__,
|
| 443 |
+
"fast_path": {
|
| 444 |
+
"flash_linear_attention": is_flash_linear_attention_available(),
|
| 445 |
+
"causal_conv1d": is_causal_conv1d_available(),
|
| 446 |
+
},
|
| 447 |
+
"linear_attn_backend": args.linear_attn_backend,
|
| 448 |
+
"vision_backend": args.vision_backend,
|
| 449 |
+
"vision_comparison": vision_comparison,
|
| 450 |
+
"text_backend": args.text_backend,
|
| 451 |
+
"text_comparison": text_comparison,
|
| 452 |
+
"forced_linear_attn_layers": forced_linear_attn_layers,
|
| 453 |
+
"shape": {
|
| 454 |
+
"records": args.records,
|
| 455 |
+
"questions": args.questions,
|
| 456 |
+
"choices": args.choices,
|
| 457 |
+
"input_tokens": [len(record.input_ids) for record in encoded],
|
| 458 |
+
"image_size": args.image_size,
|
| 459 |
+
"image_count_per_record": args.image_count if args.image_size else 0,
|
| 460 |
+
},
|
| 461 |
+
"encode_ms": encode_seconds * 1000,
|
| 462 |
+
"latency_ms": {
|
| 463 |
+
"total": summarize_ms(total_times),
|
| 464 |
+
"backbone": summarize_ms(backbone_times),
|
| 465 |
+
"head": summarize_ms(head_times),
|
| 466 |
+
},
|
| 467 |
+
"max_memory_allocated": max_memory,
|
| 468 |
+
"profile_table": profile_table,
|
| 469 |
+
"stage_profile": stage_profile,
|
| 470 |
+
}
|
| 471 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 472 |
+
args.output.write_text(json.dumps(result, indent=2) + "\n")
|
| 473 |
+
print(json.dumps({k: v for k, v in result.items() if k not in {"profile_table", "stage_profile"}}, indent=2))
|
| 474 |
+
if profile_table:
|
| 475 |
+
print(profile_table)
|
| 476 |
+
|
| 477 |
+
|
| 478 |
+
if __name__ == "__main__":
|
| 479 |
+
main()
|
runtime/clef-bootstrap-runtime
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
|
| 4 |
+
VENV="${CLEF_VENV:-/workspace/clef-venv}"
|
| 5 |
+
CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}"
|
| 6 |
+
CUDA_HOME="$(readlink -f "${CUDA_HOME}")"
|
| 7 |
+
|
| 8 |
+
export CUDA_HOME
|
| 9 |
+
export PATH="${CUDA_HOME}/bin:${PATH}"
|
| 10 |
+
export CPATH="${CUDA_HOME}/targets/x86_64-linux/include${CPATH:+:${CPATH}}"
|
| 11 |
+
export LIBRARY_PATH="${CUDA_HOME}/targets/x86_64-linux/lib/stubs${LIBRARY_PATH:+:${LIBRARY_PATH}}"
|
| 12 |
+
export TRITON_PTXAS_PATH="${TRITON_PTXAS_PATH:-${CUDA_HOME}/bin/ptxas}"
|
| 13 |
+
export TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-9.0}"
|
| 14 |
+
export MAX_JOBS="${MAX_JOBS:-8}"
|
| 15 |
+
|
| 16 |
+
python -m venv --system-site-packages "${VENV}"
|
| 17 |
+
"${VENV}/bin/python" -m pip install --upgrade pip
|
| 18 |
+
"${VENV}/bin/python" -m pip install --upgrade --no-cache-dir \
|
| 19 |
+
transformers==5.12.1 \
|
| 20 |
+
huggingface_hub \
|
| 21 |
+
safetensors \
|
| 22 |
+
pillow \
|
| 23 |
+
accelerate \
|
| 24 |
+
nvidia-cutlass-dsl==4.8.0 \
|
| 25 |
+
apache-tvm-ffi==0.1.14.post1 \
|
| 26 |
+
flash-linear-attention==0.5.2
|
| 27 |
+
"${VENV}/bin/python" -m pip install --no-cache-dir --no-build-isolation \
|
| 28 |
+
causal-conv1d==1.7.0
|
| 29 |
+
|
| 30 |
+
if [[ "${CLEF_INSTALL_INT8:-0}" == "1" ]]; then
|
| 31 |
+
"${VENV}/bin/python" -m pip install --no-cache-dir --no-deps comfy-kitchen==0.2.31
|
| 32 |
+
fi
|
| 33 |
+
|
| 34 |
+
"${VENV}/bin/python" - <<'PY'
|
| 35 |
+
import torch
|
| 36 |
+
from transformers.utils.import_utils import (
|
| 37 |
+
is_causal_conv1d_available,
|
| 38 |
+
is_flash_linear_attention_available,
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
print("torch", torch.__version__, torch.version.cuda)
|
| 42 |
+
print("flash_linear_attention", is_flash_linear_attention_available())
|
| 43 |
+
print("causal_conv1d", is_causal_conv1d_available())
|
| 44 |
+
PY
|
runtime/clef-int8-bench.py
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Bracket experimental CLEF ConvRot inference with exact fused BF16 baselines."""
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import gc
|
| 6 |
+
import importlib.util
|
| 7 |
+
import json
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
import time
|
| 10 |
+
|
| 11 |
+
spec = importlib.util.spec_from_file_location("clef_bench", Path(__file__).with_name("clef-bench.py"))
|
| 12 |
+
bench = importlib.util.module_from_spec(spec)
|
| 13 |
+
spec.loader.exec_module(bench)
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
from huggingface_hub import snapshot_download
|
| 17 |
+
from clef_text_cute import patch_text
|
| 18 |
+
from clef_text_int8 import quantize_text
|
| 19 |
+
from clef_text_metrics import decision_drift
|
| 20 |
+
from clef_text_profile import profile_text
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def semantic_records():
|
| 24 |
+
states = [
|
| 25 |
+
{"summary": "An authenticated customer requests a routine refund for a duplicate charge.",
|
| 26 |
+
"authenticated": True, "duplicate_charge": True, "amount": 24},
|
| 27 |
+
{"summary": "An unverified caller requests an urgent transfer and asks to bypass approval.",
|
| 28 |
+
"authenticated": False, "duplicate_charge": False, "amount": 95000},
|
| 29 |
+
{"summary": "A network incident is resolved. The customer confirms service has recovered.",
|
| 30 |
+
"latency_ms": 14, "packet_loss_percent": 0, "active_incident": False},
|
| 31 |
+
{"summary": "Latency is high, but the report also says the outage is resolved. Evidence conflicts.",
|
| 32 |
+
"latency_ms": 2000, "packet_loss_percent": 35, "active_incident": False},
|
| 33 |
+
]
|
| 34 |
+
return [{"id": f"semantic-{i}", "state": state, "questions": {
|
| 35 |
+
"route": {"type": "choice", "instructions": "Select the most appropriate handling route.",
|
| 36 |
+
"criteria": {"routine": "Low-risk routine support with sufficient verified evidence.",
|
| 37 |
+
"review": "Conflicting evidence or an unverified high-risk request.",
|
| 38 |
+
"incident": "An ongoing service reliability incident.",
|
| 39 |
+
"close": "A resolved issue requiring no further action."}},
|
| 40 |
+
"urgency": {"type": "choice", "instructions": "Assess urgency from evidence, not requested urgency.",
|
| 41 |
+
"criteria": {"low": "No current harm or only a routine small issue.",
|
| 42 |
+
"high": "Substantial financial risk or current severe service degradation."}},
|
| 43 |
+
}} for i, state in enumerate(states)]
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def cpu_outputs(outputs, logits):
|
| 47 |
+
return outputs.last_hidden_state.detach().cpu(), [[v.detach().float().cpu().flatten().tolist() for v in r] for r in logits]
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def hidden_error(actual, expected):
|
| 51 |
+
a, b = actual.float(), expected.float()
|
| 52 |
+
diff = a - b
|
| 53 |
+
if not torch.isfinite(a).all() or not torch.isfinite(b).all():
|
| 54 |
+
raise ValueError("Non-finite CLEF hidden states")
|
| 55 |
+
return {"max_abs": float(diff.abs().max()),
|
| 56 |
+
"relative_l2": float(diff.norm() / b.norm().clamp_min(1e-12))}
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def main():
|
| 60 |
+
p = argparse.ArgumentParser(description=__doc__)
|
| 61 |
+
p.add_argument("--model", default="Cloudflare/clef")
|
| 62 |
+
p.add_argument("--state-words", type=int, nargs="+", default=[32, 256, 1024])
|
| 63 |
+
p.add_argument("--records-file", type=Path, help="JSON array of real state/questions records; replaces synthetic probes")
|
| 64 |
+
p.add_argument("--scope", choices=["mlp-gate-up", "mlp", "all"], nargs="+", default=["mlp", "all"])
|
| 65 |
+
p.add_argument("--backend", choices=["cute", "torch"], default="cute")
|
| 66 |
+
p.add_argument("--calibration-index", type=Path, help="Indexed calibrated ConvRot linear overrides")
|
| 67 |
+
p.add_argument("--tile", type=int, nargs=2, default=[128, 256])
|
| 68 |
+
p.add_argument("--cluster", type=int, nargs=2, default=[1, 1])
|
| 69 |
+
p.add_argument("--heads", action="store_true", help="Include experimental exact 256-wide Q/K norm fusion")
|
| 70 |
+
p.add_argument("--share-mlp-input", action="store_true", help="Rotate/quantize the gate/up input only once")
|
| 71 |
+
p.add_argument("--share-attn-input", action="store_true", help="Rotate/quantize the GatedDeltaNet QKV/Z input only once")
|
| 72 |
+
p.add_argument("--warmup", type=int, default=3)
|
| 73 |
+
p.add_argument("--repeat", type=int, default=30)
|
| 74 |
+
p.add_argument("--max-length", type=int, default=16384)
|
| 75 |
+
p.add_argument("--profile", action="store_true")
|
| 76 |
+
p.add_argument("--capture-nsys", action="store_true")
|
| 77 |
+
p.add_argument("--profile-mode", choices=["bf16_before", "mlp-gate-up", "mlp", "all", "bf16_after"], default="all")
|
| 78 |
+
p.add_argument("--profile-case", type=int, default=1)
|
| 79 |
+
p.add_argument("--profile-iterations", type=int, default=3)
|
| 80 |
+
p.add_argument("--output", type=Path, default=Path("clef-int8-bench.json"))
|
| 81 |
+
args = p.parse_args()
|
| 82 |
+
if args.repeat < 1 or args.warmup < 1 or args.profile_iterations < 1:
|
| 83 |
+
p.error("Repeat, warmup, and profile iterations must be positive")
|
| 84 |
+
path = Path(args.model)
|
| 85 |
+
if not path.is_dir():
|
| 86 |
+
path = Path(snapshot_download(args.model, local_files_only=True))
|
| 87 |
+
joint = bench.load_joint_schema_module(path)
|
| 88 |
+
device = torch.device("cuda")
|
| 89 |
+
if torch.cuda.get_device_capability(device) != (9, 0):
|
| 90 |
+
raise ValueError("This experiment requires a Hopper SM90 GPU")
|
| 91 |
+
if args.records_file:
|
| 92 |
+
records = json.loads(args.records_file.read_text())
|
| 93 |
+
if not isinstance(records, list) or not records or any(
|
| 94 |
+
not isinstance(r, dict) or "state" not in r or "questions" not in r for r in records):
|
| 95 |
+
raise ValueError("Expected a nonempty JSON array of state/questions records")
|
| 96 |
+
else:
|
| 97 |
+
records = [bench.make_record(i, 8, 4, words) for i, words in enumerate(args.state_words)] + semantic_records()
|
| 98 |
+
if (args.profile or args.capture_nsys) and not 0 <= args.profile_case < len(records):
|
| 99 |
+
p.error("Profile case must select an existing record")
|
| 100 |
+
modes = [scope for scope in ("mlp-gate-up", "mlp", "all") if scope in args.scope]
|
| 101 |
+
if args.calibration_index is not None and len(modes) != 1:
|
| 102 |
+
p.error("A calibrated checkpoint requires exactly one matching --scope")
|
| 103 |
+
if args.calibration_index is not None:
|
| 104 |
+
from clef_int8_calibration import source_digest
|
| 105 |
+
metadata = json.loads(args.calibration_index.read_text())["metadata"]
|
| 106 |
+
if metadata.get("source_digest") != source_digest(path):
|
| 107 |
+
p.error("Calibrated weights were made from a different model/config/code manifest")
|
| 108 |
+
if (args.profile or args.capture_nsys) and args.profile_mode not in {"bf16_before", "bf16_after", *modes}:
|
| 109 |
+
p.error("Profile mode must be selected by --scope")
|
| 110 |
+
result = {"model_path": str(path), "torch": torch.__version__, "device": torch.cuda.get_device_name(),
|
| 111 |
+
"backend": args.backend, "heads": args.heads, "share_mlp_input": args.share_mlp_input,
|
| 112 |
+
"share_attn_input": args.share_attn_input, "modes": {},
|
| 113 |
+
"quality_scope": "BF16 agreement only, not task accuracy or production validation",
|
| 114 |
+
"records_source": "external" if args.records_file else "synthetic", "bracketed": False}
|
| 115 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 116 |
+
|
| 117 |
+
def save():
|
| 118 |
+
args.output.write_text(json.dumps(result, indent=2) + "\n")
|
| 119 |
+
|
| 120 |
+
with torch.inference_mode():
|
| 121 |
+
model, processor = joint.load_release_model(path, device=device, dtype=torch.bfloat16)
|
| 122 |
+
batches, labels = [], []
|
| 123 |
+
for i, record in enumerate(records):
|
| 124 |
+
encoded = joint.encode_record(processor.tokenizer, record, processor=processor, max_length=args.max_length)
|
| 125 |
+
batches.append(joint.collate_records([encoded], processor.tokenizer.pad_token_id, device))
|
| 126 |
+
labels.append({"case": i, "tokens": len(encoded.input_ids)})
|
| 127 |
+
references = []
|
| 128 |
+
|
| 129 |
+
def components(model):
|
| 130 |
+
base = model.language_model.get_base_model() if hasattr(model.language_model, "get_base_model") else model.language_model
|
| 131 |
+
return base, getattr(base.model, "language_model", base.model)
|
| 132 |
+
|
| 133 |
+
def run_once(model, base, text, batch):
|
| 134 |
+
outputs = text(input_ids=batch["input_ids"], attention_mask=batch["attention_mask"],
|
| 135 |
+
use_cache=False, return_dict=True)
|
| 136 |
+
logits = model.head(outputs.last_hidden_state, batch["input_ids"], batch["attention_mask"],
|
| 137 |
+
batch["records"], base.get_output_embeddings().weight)
|
| 138 |
+
return outputs, logits
|
| 139 |
+
|
| 140 |
+
base, text = components(model)
|
| 141 |
+
for batch in batches:
|
| 142 |
+
references.append(cpu_outputs(*run_once(model, base, text, batch)))
|
| 143 |
+
restore = patch_text(text, gva=True, gates=True, heads=args.heads, share_mlp_quant=args.share_mlp_input,
|
| 144 |
+
share_attn_quant=args.share_attn_input)
|
| 145 |
+
|
| 146 |
+
def measure_mode(label, conversion=None, quant_references=None):
|
| 147 |
+
rows = []
|
| 148 |
+
for i, batch in enumerate(batches):
|
| 149 |
+
fn = lambda batch=batch: run_once(model, base, text, batch)
|
| 150 |
+
hidden, logits = cpu_outputs(*fn())
|
| 151 |
+
if quant_references is not None:
|
| 152 |
+
quant_hidden, quant_logits = quant_references[i]
|
| 153 |
+
torch.testing.assert_close(hidden, quant_hidden, atol=0, rtol=0)
|
| 154 |
+
if decision_drift(logits, quant_logits)["logits_max_abs"] != 0:
|
| 155 |
+
raise AssertionError("Shared MLP input changed quantized CLEF logits")
|
| 156 |
+
ref_hidden, ref_logits = references[i]
|
| 157 |
+
drift = decision_drift(logits, ref_logits)
|
| 158 |
+
if label.startswith("bf16"):
|
| 159 |
+
torch.testing.assert_close(hidden, ref_hidden, atol=0, rtol=0)
|
| 160 |
+
if drift["logits_max_abs"] != 0:
|
| 161 |
+
raise AssertionError("Exact text fusions changed CLEF logits")
|
| 162 |
+
error = hidden_error(hidden, ref_hidden)
|
| 163 |
+
del hidden
|
| 164 |
+
for _ in range(args.warmup):
|
| 165 |
+
fn()
|
| 166 |
+
torch.cuda.synchronize()
|
| 167 |
+
torch.cuda.reset_peak_memory_stats()
|
| 168 |
+
times = [bench.timed(fn, device)[1] for _ in range(args.repeat)]
|
| 169 |
+
backbone_fn = lambda batch=batch: text(input_ids=batch["input_ids"], attention_mask=batch["attention_mask"],
|
| 170 |
+
use_cache=False, return_dict=True)
|
| 171 |
+
backbone = [bench.timed(backbone_fn, device)[1] for _ in range(args.repeat)]
|
| 172 |
+
row = {**labels[i], "total_ms": bench.summarize_ms(times),
|
| 173 |
+
"backbone_ms": bench.summarize_ms(backbone), "hidden_error": error, "drift": drift,
|
| 174 |
+
"resident_vram_bytes": torch.cuda.memory_allocated(),
|
| 175 |
+
"peak_vram_bytes": torch.cuda.max_memory_allocated()}
|
| 176 |
+
rows.append(row)
|
| 177 |
+
print(json.dumps({"mode": label, "case": i, "total_ms": row["total_ms"]["median"],
|
| 178 |
+
"matched": drift["decisions_matched"], "decisions": drift["decisions_total"],
|
| 179 |
+
"logits_max_abs": drift["logits_max_abs"]}), flush=True)
|
| 180 |
+
result["modes"][label] = {"cases": rows, "conversion": conversion,
|
| 181 |
+
"shared_input_exact": True if quant_references is not None else None,
|
| 182 |
+
"patch_statistics": dict(restore.statistics)}
|
| 183 |
+
save()
|
| 184 |
+
if label == args.profile_mode and (args.profile or args.capture_nsys):
|
| 185 |
+
batch = batches[args.profile_case]
|
| 186 |
+
result["modes"][label]["profile"] = profile_text(
|
| 187 |
+
lambda: run_once(model, base, text, batch), text,
|
| 188 |
+
args.output.parent / (args.output.stem + "-" + label + "-profile"), head=model.head,
|
| 189 |
+
iterations=args.profile_iterations, capture_nsys=args.capture_nsys)
|
| 190 |
+
save()
|
| 191 |
+
|
| 192 |
+
measure_mode("bf16_before")
|
| 193 |
+
for scope in modes:
|
| 194 |
+
torch.cuda.synchronize()
|
| 195 |
+
started = time.perf_counter()
|
| 196 |
+
conversion = quantize_text(text, scope=scope, backend=args.backend, tile=args.tile, cluster=args.cluster,
|
| 197 |
+
calibration_index=args.calibration_index)
|
| 198 |
+
conversion["seconds"] = time.perf_counter() - started
|
| 199 |
+
quant_references = None
|
| 200 |
+
if args.share_mlp_input or args.share_attn_input:
|
| 201 |
+
restore()
|
| 202 |
+
del restore
|
| 203 |
+
unshared = patch_text(text, gva=True, gates=True, heads=args.heads)
|
| 204 |
+
try:
|
| 205 |
+
quant_references = [cpu_outputs(*run_once(model, base, text, batch)) for batch in batches]
|
| 206 |
+
finally:
|
| 207 |
+
unshared()
|
| 208 |
+
del unshared
|
| 209 |
+
restore = patch_text(text, gva=True, gates=True, heads=args.heads,
|
| 210 |
+
share_mlp_quant=args.share_mlp_input, share_attn_quant=args.share_attn_input)
|
| 211 |
+
measure_mode(scope, conversion, quant_references)
|
| 212 |
+
# Release original forward closures before reloading, rather than retaining two full models.
|
| 213 |
+
restore()
|
| 214 |
+
del restore, model, base, text, processor
|
| 215 |
+
gc.collect()
|
| 216 |
+
torch.cuda.empty_cache()
|
| 217 |
+
model, processor = joint.load_release_model(path, device=device, dtype=torch.bfloat16)
|
| 218 |
+
base, text = components(model)
|
| 219 |
+
restore = patch_text(text, gva=True, gates=True, heads=args.heads, share_mlp_quant=args.share_mlp_input,
|
| 220 |
+
share_attn_quant=args.share_attn_input)
|
| 221 |
+
try:
|
| 222 |
+
measure_mode("bf16_after")
|
| 223 |
+
finally:
|
| 224 |
+
restore()
|
| 225 |
+
for scope in modes:
|
| 226 |
+
for before, candidate, after in zip(result["modes"]["bf16_before"]["cases"],
|
| 227 |
+
result["modes"][scope]["cases"],
|
| 228 |
+
result["modes"]["bf16_after"]["cases"], strict=True):
|
| 229 |
+
reference_ms = (before["total_ms"]["median"] + after["total_ms"]["median"]) / 2
|
| 230 |
+
candidate["speedup_vs_fused_bf16"] = reference_ms / candidate["total_ms"]["median"]
|
| 231 |
+
result["bracketed"] = True
|
| 232 |
+
save()
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
if __name__ == "__main__":
|
| 236 |
+
main()
|
runtime/clef-int8-calibrate.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Calibrate ConvRot weights, export indexed shards, and evaluate untouched records."""
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import gc
|
| 6 |
+
import importlib.util
|
| 7 |
+
import json
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
import random
|
| 10 |
+
import time
|
| 11 |
+
|
| 12 |
+
spec = importlib.util.spec_from_file_location("clef_int8_bench", Path(__file__).with_name("clef-int8-bench.py"))
|
| 13 |
+
helpers = importlib.util.module_from_spec(spec)
|
| 14 |
+
spec.loader.exec_module(helpers)
|
| 15 |
+
bench = helpers.bench
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
from huggingface_hub import snapshot_download
|
| 19 |
+
from clef_text_cute import patch_text
|
| 20 |
+
from clef_text_int8 import quantize_text
|
| 21 |
+
from clef_text_metrics import decision_drift
|
| 22 |
+
from clef_int8_calibration import (CalibrationWriter, calibrate_linear, capture_samples,
|
| 23 |
+
projection_names, records_digest, source_digest)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def corpus(seed, count):
|
| 27 |
+
rng = random.Random(seed)
|
| 28 |
+
topics = ["duplicate charge", "failed delivery", "service outage", "suspicious transfer",
|
| 29 |
+
"contract renewal", "account access", "payment dispute", "network latency"]
|
| 30 |
+
return [{"id": f"cal-{seed}-{i}", "state": {
|
| 31 |
+
"case_id": f"case-{seed}-{i}", "request": topics[i % len(topics)],
|
| 32 |
+
"summary": " ".join(rng.choice([
|
| 33 |
+
"The customer confirmed the incident is resolved.", "The identity check has not been completed.",
|
| 34 |
+
"The evidence is contradictory and needs independent review.", "An automated alert indicates ongoing harm.",
|
| 35 |
+
"The request follows the standard policy and approval process.", "The requester wants to bypass approval.",
|
| 36 |
+
"The record includes verified supporting documentation.", "No evidence was provided for the claim."
|
| 37 |
+
]) for _ in range(rng.choice([1, 3, 8, 24]))),
|
| 38 |
+
"amount": rng.choice([0, 8, 99, 1500, 18000, 125000]),
|
| 39 |
+
"authenticated": bool(rng.getrandbits(1)), "risk_score": round(rng.random(), 3),
|
| 40 |
+
"latency_ms": rng.choice([8, 95, 700, 3000]), "active_incident": bool(rng.getrandbits(1)),
|
| 41 |
+
}, "questions": {
|
| 42 |
+
"route": {"type": "choice", "instructions": "Choose the handling route using the available evidence.",
|
| 43 |
+
"criteria": {"routine": "A verified, low-risk routine support request.",
|
| 44 |
+
"review": "Contradictory or incomplete evidence.",
|
| 45 |
+
"escalate": "Substantial current financial risk or active service harm.",
|
| 46 |
+
"close": "The issue is resolved and no further action is necessary."}},
|
| 47 |
+
"urgency": {"type": "choice", "instructions": "Determine urgency from actual harm rather than requested urgency.",
|
| 48 |
+
"criteria": {"low": "Resolved or minor routine issue.", "medium": "Unresolved but contained problem.",
|
| 49 |
+
"high": "Severe ongoing harm or substantial financial exposure."}},
|
| 50 |
+
"verification": {"type": "choice", "instructions": "Determine whether the evidence is sufficient to proceed.",
|
| 51 |
+
"criteria": {"sufficient": "Verified identity and consistent supporting evidence.",
|
| 52 |
+
"insufficient": "Unverified identity, missing evidence, or conflicting facts."}},
|
| 53 |
+
}} for i in range(count)]
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def read_records(path, seed, count):
|
| 57 |
+
records = json.loads(path.read_text()) if path else corpus(seed, count)
|
| 58 |
+
if (not isinstance(records, list) or not records or any(not isinstance(r, dict)
|
| 59 |
+
or "state" not in r or "questions" not in r or r.get("images") for r in records)):
|
| 60 |
+
raise ValueError("Expected a nonempty array of text state/questions records")
|
| 61 |
+
return records
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def input_digest(record):
|
| 65 |
+
# IDs are bookkeeping, not evidence of distinct model inputs.
|
| 66 |
+
return records_digest([{key: record[key] for key in ("state", "questions")}])
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def main():
|
| 70 |
+
p = argparse.ArgumentParser(description=__doc__)
|
| 71 |
+
p.add_argument("--model", default="Cloudflare/clef")
|
| 72 |
+
p.add_argument("--scope", choices=["mlp-gate-up", "mlp", "all"], default="all")
|
| 73 |
+
p.add_argument("--calibration-records", type=Path)
|
| 74 |
+
p.add_argument("--selection-records", type=Path)
|
| 75 |
+
p.add_argument("--heldout-records", type=Path)
|
| 76 |
+
p.add_argument("--train-count", type=int, default=12)
|
| 77 |
+
p.add_argument("--selection-count", type=int, default=6)
|
| 78 |
+
p.add_argument("--heldout-count", type=int, default=16)
|
| 79 |
+
p.add_argument("--sample-rows", type=int, default=16)
|
| 80 |
+
p.add_argument("--clips", type=float, nargs="+", default=[1.0, 0.99, 0.97, 0.94, 0.90])
|
| 81 |
+
p.add_argument("--repeat", type=int, default=30)
|
| 82 |
+
p.add_argument("--weights-dir", type=Path, required=True)
|
| 83 |
+
p.add_argument("--output", type=Path, default=Path("clef-int8-calibration.json"))
|
| 84 |
+
args = p.parse_args()
|
| 85 |
+
if min(args.repeat, args.sample_rows, args.train_count, args.selection_count, args.heldout_count) < 1:
|
| 86 |
+
p.error("Counts and repeats must be positive")
|
| 87 |
+
records = [read_records(args.calibration_records, 101, args.train_count),
|
| 88 |
+
read_records(args.selection_records, 211, args.selection_count),
|
| 89 |
+
read_records(args.heldout_records, 4242, args.heldout_count)]
|
| 90 |
+
hashes = [{input_digest(r) for r in group} for group in records]
|
| 91 |
+
if any(hashes[i] & hashes[j] for i in range(3) for j in range(i + 1, 3)):
|
| 92 |
+
raise ValueError("Calibration, selection, and held-out records must not overlap")
|
| 93 |
+
path = Path(args.model)
|
| 94 |
+
if not path.is_dir():
|
| 95 |
+
path = Path(snapshot_download(args.model, local_files_only=True))
|
| 96 |
+
joint = bench.load_joint_schema_module(path)
|
| 97 |
+
device = torch.device("cuda")
|
| 98 |
+
result = {"source_digest": source_digest(path), "model_path": str(path),
|
| 99 |
+
"torch": torch.__version__, "device": torch.cuda.get_device_name(),
|
| 100 |
+
"scope": args.scope, "group_size": 256,
|
| 101 |
+
"calibration_digest": records_digest(records[0]), "selection_digest": records_digest(records[1]),
|
| 102 |
+
"heldout_digest": records_digest(records[2]), "records": [len(r) for r in records],
|
| 103 |
+
"data_source": "external" if any([args.calibration_records, args.selection_records, args.heldout_records]) else "synthetic",
|
| 104 |
+
"split_sources": {split: "external" if filename else "synthetic" for split, filename in zip(
|
| 105 |
+
("calibration", "selection", "heldout"),
|
| 106 |
+
(args.calibration_records, args.selection_records, args.heldout_records), strict=True)},
|
| 107 |
+
"calibration_config": {"clips": args.clips, "sample_rows_per_record": args.sample_rows,
|
| 108 |
+
"ridge": 0.05, "max_gain": 0.03, "min_improvement": 0.005,
|
| 109 |
+
"tile": [128, 256], "cluster": [2, 1]},
|
| 110 |
+
"method": "per-output-row clipping, bounded ridge scale/bias fit, independent local selection",
|
| 111 |
+
"quality_scope": "BF16 agreement, not production task accuracy", "linears": {}, "evaluation": {}}
|
| 112 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 113 |
+
def save():
|
| 114 |
+
args.output.write_text(json.dumps(result, indent=2) + "\n")
|
| 115 |
+
with torch.inference_mode():
|
| 116 |
+
model, processor = joint.load_release_model(path, device=device, dtype=torch.bfloat16)
|
| 117 |
+
def components(model):
|
| 118 |
+
base = model.language_model.get_base_model() if hasattr(model.language_model, "get_base_model") else model.language_model
|
| 119 |
+
return base, getattr(base.model, "language_model", base.model)
|
| 120 |
+
base, text = components(model)
|
| 121 |
+
restore = patch_text(text, gva=True, gates=True, heads=True, share_mlp_quant=True, share_attn_quant=True)
|
| 122 |
+
def encode(group):
|
| 123 |
+
return [joint.collate_records([joint.encode_record(processor.tokenizer, r, processor=processor, max_length=16384)],
|
| 124 |
+
processor.tokenizer.pad_token_id, device) for r in group]
|
| 125 |
+
train_batches, selection_batches, heldout_batches = [encode(group) for group in records]
|
| 126 |
+
regression_records = [bench.make_record(i, 8, 4, words) for i, words in enumerate([32, 256, 1024])] + helpers.semantic_records()
|
| 127 |
+
regression_batches = encode(regression_records)
|
| 128 |
+
all_eval_batches = heldout_batches + regression_batches
|
| 129 |
+
def backbone(batch):
|
| 130 |
+
return text(input_ids=batch["input_ids"], attention_mask=batch["attention_mask"], use_cache=False, return_dict=True)
|
| 131 |
+
def run(batch):
|
| 132 |
+
outputs = backbone(batch)
|
| 133 |
+
logits = model.head(outputs.last_hidden_state, batch["input_ids"], batch["attention_mask"],
|
| 134 |
+
batch["records"], base.get_output_embeddings().weight)
|
| 135 |
+
return outputs, logits
|
| 136 |
+
references = [helpers.cpu_outputs(*run(batch)) for batch in all_eval_batches]
|
| 137 |
+
def evaluate(label):
|
| 138 |
+
rows = []
|
| 139 |
+
for i, batch in enumerate(all_eval_batches):
|
| 140 |
+
hidden, logits = helpers.cpu_outputs(*run(batch))
|
| 141 |
+
expected_hidden, expected_logits = references[i]
|
| 142 |
+
rows.append({"case": i, "split": "heldout" if i < len(heldout_batches) else "regression",
|
| 143 |
+
"tokens": batch["input_ids"].shape[1], "hidden_error": helpers.hidden_error(hidden, expected_hidden),
|
| 144 |
+
"drift": decision_drift(logits, expected_logits)})
|
| 145 |
+
timing_batch = regression_batches[1]
|
| 146 |
+
for _ in range(3):
|
| 147 |
+
run(timing_batch)
|
| 148 |
+
torch.cuda.reset_peak_memory_stats()
|
| 149 |
+
times = [bench.timed(lambda: run(timing_batch), device)[1] for _ in range(args.repeat)]
|
| 150 |
+
result["evaluation"][label] = {"cases": rows, "total_ms": bench.summarize_ms(times),
|
| 151 |
+
"resident_vram_bytes": torch.cuda.memory_allocated(),
|
| 152 |
+
"peak_vram_bytes": torch.cuda.max_memory_allocated()}
|
| 153 |
+
print(json.dumps({"mode": label, "ms": result["evaluation"][label]["total_ms"]["median"],
|
| 154 |
+
"heldout_matched": sum(r["drift"]["decisions_matched"] for r in rows if r["split"] == "heldout"),
|
| 155 |
+
"heldout_total": sum(r["drift"]["decisions_total"] for r in rows if r["split"] == "heldout"),
|
| 156 |
+
"regression_matched": sum(r["drift"]["decisions_matched"] for r in rows if r["split"] == "regression")}), flush=True)
|
| 157 |
+
save()
|
| 158 |
+
evaluate("bf16")
|
| 159 |
+
names = projection_names(text, args.scope)
|
| 160 |
+
train = capture_samples(text, train_batches, backbone, names, args.sample_rows)
|
| 161 |
+
selection = capture_samples(text, selection_batches, backbone, names, args.sample_rows)
|
| 162 |
+
quantize_text(text, scope=args.scope, cluster=(2, 1))
|
| 163 |
+
evaluate("uncalibrated")
|
| 164 |
+
restore()
|
| 165 |
+
del restore, model, base, text, processor
|
| 166 |
+
gc.collect()
|
| 167 |
+
torch.cuda.empty_cache()
|
| 168 |
+
model, processor = joint.load_release_model(path, device=device, dtype=torch.bfloat16)
|
| 169 |
+
base, text = components(model)
|
| 170 |
+
restore = patch_text(text, gva=True, gates=True, heads=True, share_mlp_quant=True, share_attn_quant=True)
|
| 171 |
+
writer = CalibrationWriter(args.weights_dir, {key: result[key] for key in (
|
| 172 |
+
"source_digest", "model_path", "scope", "calibration_digest", "selection_digest", "method",
|
| 173 |
+
"data_source", "split_sources", "calibration_config")})
|
| 174 |
+
started = time.perf_counter()
|
| 175 |
+
for i, name in enumerate(names):
|
| 176 |
+
parent_name, leaf = name.rsplit(".", 1)
|
| 177 |
+
parent = text.get_submodule(parent_name)
|
| 178 |
+
linear = getattr(parent, leaf)
|
| 179 |
+
calibrated, stats = calibrate_linear(linear, train.pop(name), selection.pop(name), clips=args.clips)
|
| 180 |
+
setattr(parent, leaf, calibrated)
|
| 181 |
+
del linear
|
| 182 |
+
writer.add(name, calibrated)
|
| 183 |
+
result["linears"][name] = stats
|
| 184 |
+
if i % 16 == 0 or i == len(names) - 1:
|
| 185 |
+
print(json.dumps({"linear": i + 1, "total": len(names), "name": name,
|
| 186 |
+
"selection_mse_ratio": stats["selection_mse_ratio"]}), flush=True)
|
| 187 |
+
save()
|
| 188 |
+
result["calibration_seconds"] = time.perf_counter() - started
|
| 189 |
+
result["calibration_index"] = str(writer.finish())
|
| 190 |
+
del writer, calibrated, parent
|
| 191 |
+
torch.cuda.empty_cache()
|
| 192 |
+
evaluate("calibrated")
|
| 193 |
+
expected_reload = [helpers.cpu_outputs(*run(batch)) for batch in all_eval_batches]
|
| 194 |
+
restore()
|
| 195 |
+
del restore, model, base, text, processor
|
| 196 |
+
gc.collect()
|
| 197 |
+
torch.cuda.empty_cache()
|
| 198 |
+
model, processor = joint.load_release_model(path, device=device, dtype=torch.bfloat16)
|
| 199 |
+
base, text = components(model)
|
| 200 |
+
quantize_text(text, scope=args.scope, cluster=(2, 1), calibration_index=Path(result["calibration_index"]))
|
| 201 |
+
restore = patch_text(text, gva=True, gates=True, heads=True, share_mlp_quant=True, share_attn_quant=True)
|
| 202 |
+
try:
|
| 203 |
+
for batch, (expected_hidden, expected_logits) in zip(all_eval_batches, expected_reload, strict=True):
|
| 204 |
+
hidden, logits = helpers.cpu_outputs(*run(batch))
|
| 205 |
+
torch.testing.assert_close(hidden, expected_hidden, atol=0, rtol=0)
|
| 206 |
+
if decision_drift(logits, expected_logits)["logits_max_abs"] != 0:
|
| 207 |
+
raise AssertionError("Indexed calibrated checkpoint changed logits after reload")
|
| 208 |
+
result["indexed_reload_exact"] = True
|
| 209 |
+
finally:
|
| 210 |
+
restore()
|
| 211 |
+
save()
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
if __name__ == "__main__":
|
| 215 |
+
main()
|
runtime/clef-int8-calibration-check.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Check FP32 ConvRot calibration and CuTe epilogue/reload parity on H100."""
|
| 3 |
+
|
| 4 |
+
import json
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
import tempfile
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
from clef_int8_calibration import CalibrationIndex, CalibrationWriter, calibrate_linear, integer_projection
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def main():
|
| 14 |
+
torch.manual_seed(42)
|
| 15 |
+
with torch.inference_mode():
|
| 16 |
+
text = torch.nn.Module().cuda().eval()
|
| 17 |
+
text.proj = torch.nn.Linear(1024, 1024, bias=False, device="cuda", dtype=torch.bfloat16).eval()
|
| 18 |
+
def samples(rows):
|
| 19 |
+
x = torch.randn(rows, 1024, device="cuda", dtype=torch.bfloat16)
|
| 20 |
+
return x.cpu(), text.proj(x).cpu()
|
| 21 |
+
module, stats = calibrate_linear(text.proj, samples(128), samples(64))
|
| 22 |
+
assert stats["selection_mse_after"] <= stats["selection_mse_before"]
|
| 23 |
+
assert module.bias.count_nonzero() > 0
|
| 24 |
+
x = torch.randn(33, 1024, device="cuda", dtype=torch.bfloat16)
|
| 25 |
+
q, s = module.quantize_input(x)
|
| 26 |
+
expected = (integer_projection(q, s, module.weight) * module.weight_scale + module.bias.float()).to(torch.bfloat16)
|
| 27 |
+
torch.testing.assert_close(module(x), expected, rtol=0, atol=0)
|
| 28 |
+
with tempfile.TemporaryDirectory() as tmp:
|
| 29 |
+
writer = CalibrationWriter(Path(tmp), {"scope": "all"})
|
| 30 |
+
writer.add("proj", module)
|
| 31 |
+
index = CalibrationIndex(writer.finish(), text, "all")
|
| 32 |
+
loaded = index.load("proj", text.proj, backend="cute", tile=(128, 256), cluster=(2, 1))
|
| 33 |
+
torch.testing.assert_close(loaded(x), expected, rtol=0, atol=0)
|
| 34 |
+
stream = torch.cuda.Stream()
|
| 35 |
+
stream.wait_stream(torch.cuda.current_stream())
|
| 36 |
+
with torch.cuda.stream(stream):
|
| 37 |
+
streamed = loaded(x)
|
| 38 |
+
torch.cuda.current_stream().wait_stream(stream)
|
| 39 |
+
torch.testing.assert_close(streamed, expected, rtol=0, atol=0)
|
| 40 |
+
static = x.clone()
|
| 41 |
+
for _ in range(3):
|
| 42 |
+
loaded(static)
|
| 43 |
+
torch.cuda.synchronize()
|
| 44 |
+
graph = torch.cuda.CUDAGraph()
|
| 45 |
+
with torch.cuda.graph(graph):
|
| 46 |
+
captured = loaded(static)
|
| 47 |
+
changed = torch.randn_like(x)
|
| 48 |
+
static.copy_(changed)
|
| 49 |
+
graph.replay()
|
| 50 |
+
torch.testing.assert_close(captured, loaded(changed), rtol=0, atol=0)
|
| 51 |
+
print(json.dumps({"fp32_rotation": True, "cute_epilogue_exact": True, "reload_exact": True,
|
| 52 |
+
"nondefault_stream_exact": True, "changed_input_graph_exact": True, **stats}))
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
if __name__ == "__main__":
|
| 56 |
+
main()
|
runtime/clef_int8_calibration.py
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Activation-reconstruction calibration for the existing CLEF W8A8 runtime."""
|
| 2 |
+
|
| 3 |
+
import hashlib
|
| 4 |
+
import json
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
from safetensors import safe_open
|
| 9 |
+
from safetensors.torch import save_file
|
| 10 |
+
|
| 11 |
+
from clef_text_int8 import ConvRotLinear
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def projection_names(text, scope):
|
| 15 |
+
if scope not in {"mlp-gate-up", "mlp", "all"}:
|
| 16 |
+
raise ValueError("Unknown calibration scope")
|
| 17 |
+
return [name for name, module in text.named_modules()
|
| 18 |
+
if isinstance(module, torch.nn.Linear) and min(module.in_features, module.out_features) >= 1024
|
| 19 |
+
and (scope == "all" or ".mlp." in name)
|
| 20 |
+
and (scope != "mlp-gate-up" or name.rsplit(".", 1)[-1] in {"gate_proj", "up_proj"})]
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def records_digest(records):
|
| 24 |
+
return hashlib.sha256(json.dumps(records, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def source_digest(path):
|
| 28 |
+
digest = hashlib.sha256()
|
| 29 |
+
for name in ("config.json", "model.safetensors.index.json", "joint_schema_model.py"):
|
| 30 |
+
digest.update(name.encode())
|
| 31 |
+
digest.update((path / name).read_bytes())
|
| 32 |
+
return digest.hexdigest()
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def capture_samples(text, batches, run, names, rows_per_record=16):
|
| 36 |
+
"""Bounded BF16 CPU samples, not full intermediate activations or a state dict."""
|
| 37 |
+
if rows_per_record < 1:
|
| 38 |
+
raise ValueError("Rows per record must be positive")
|
| 39 |
+
collected = {name: ([], []) for name in names}
|
| 40 |
+
handles = []
|
| 41 |
+
for name in names:
|
| 42 |
+
def hook(module, inputs, output, name=name):
|
| 43 |
+
x = inputs[0].reshape(-1, module.in_features)
|
| 44 |
+
y = output.reshape(-1, module.out_features)
|
| 45 |
+
count = min(rows_per_record, x.shape[0])
|
| 46 |
+
indices = torch.linspace(0, x.shape[0] - 1, count, device=x.device).long()
|
| 47 |
+
collected[name][0].append(x.index_select(0, indices).cpu())
|
| 48 |
+
collected[name][1].append(y.index_select(0, indices).cpu())
|
| 49 |
+
handles.append(text.get_submodule(name).register_forward_hook(hook))
|
| 50 |
+
try:
|
| 51 |
+
for batch in batches:
|
| 52 |
+
run(batch)
|
| 53 |
+
finally:
|
| 54 |
+
for handle in handles:
|
| 55 |
+
handle.remove()
|
| 56 |
+
return {name: (torch.cat(x), torch.cat(y)) for name, (x, y) in collected.items()}
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def fit_affine(z, target, *, affine, ridge=0.05, max_gain=0.03):
|
| 60 |
+
"""Fit a bounded gain, shrunk towards one; optionally fit a shrunk offset."""
|
| 61 |
+
if z.shape != target.shape or z.ndim != 2 or not 0 <= max_gain < 1 or ridge < 0:
|
| 62 |
+
raise ValueError("Invalid calibration regression")
|
| 63 |
+
power = z.square().mean(0)
|
| 64 |
+
cross = (z * target).mean(0)
|
| 65 |
+
mean_z, mean_y = z.mean(0), target.mean(0)
|
| 66 |
+
if affine:
|
| 67 |
+
cross = cross - mean_z * mean_y
|
| 68 |
+
variance = (power - mean_z.square()).clamp_min(0)
|
| 69 |
+
else:
|
| 70 |
+
variance = power
|
| 71 |
+
gain = (cross + ridge * power) / (variance + ridge * power).clamp_min(1e-20)
|
| 72 |
+
gain = torch.where(power > 1e-20, gain, torch.ones_like(gain)).clamp(1 - max_gain, 1 + max_gain)
|
| 73 |
+
offset = (mean_y - mean_z * gain) / (1 + ridge) if affine else torch.zeros_like(gain)
|
| 74 |
+
return gain, offset
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def integer_projection(qx, xs, qw):
|
| 78 |
+
padding = max(24, (qx.shape[0] + 7) // 8 * 8) - qx.shape[0]
|
| 79 |
+
padded = torch.nn.functional.pad(qx, (0, 0, 0, padding)) if padding else qx
|
| 80 |
+
return torch._int_mm(padded, qw.t())[:qx.shape[0]].float() * xs
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def calibrate_linear(linear, train, selection, *, clips=(1.0, 0.99, 0.97, 0.94, 0.90),
|
| 84 |
+
block_rows=512, min_improvement=0.005, ridge=0.05, max_gain=0.03,
|
| 85 |
+
tile=(128, 256), cluster=(2, 1)):
|
| 86 |
+
from comfy_kitchen import _cuda_backend as cuda
|
| 87 |
+
|
| 88 |
+
if torch.is_grad_enabled() or linear.training or block_rows < 8 or block_rows % 8:
|
| 89 |
+
raise ValueError("Calibrate evaluated weights in inference mode with aligned row blocks")
|
| 90 |
+
if not clips or any(not 0 < c <= 1 for c in clips) or not 0 <= min_improvement < 1:
|
| 91 |
+
raise ValueError("Invalid clipping or improvement threshold")
|
| 92 |
+
module = ConvRotLinear(linear, tile=tile, cluster=cluster)
|
| 93 |
+
for samples in (train, selection):
|
| 94 |
+
if (len(samples) != 2 or samples[0].ndim != 2 or samples[1].ndim != 2
|
| 95 |
+
or samples[0].shape[0] != samples[1].shape[0] or not samples[0].shape[0]
|
| 96 |
+
or samples[0].shape[1] != linear.in_features or samples[1].shape[1] != linear.out_features):
|
| 97 |
+
raise ValueError("Invalid calibration sample shapes")
|
| 98 |
+
tx, ty = [v.to(linear.weight.device) for v in train]
|
| 99 |
+
vx, vy = [v.to(linear.weight.device) for v in selection]
|
| 100 |
+
tq, ts = module.quantize_input(tx)
|
| 101 |
+
vq, vs = module.quantize_input(vx)
|
| 102 |
+
ty, vy = ty.float(), vy.float()
|
| 103 |
+
if not all(torch.isfinite(v).all() for v in (ty, vy, tx, vx)):
|
| 104 |
+
raise ValueError("Non-finite calibration samples")
|
| 105 |
+
# The runtime's fused quantizer rotates in FP32 before INT8 rounding.
|
| 106 |
+
rotated = cuda.rotate_int8_convrot_weight(linear.weight.detach().float().contiguous(), 256)
|
| 107 |
+
selected, before, after = [], [], []
|
| 108 |
+
for start in range(0, module.out_features, block_rows):
|
| 109 |
+
stop = min(module.out_features, start + block_rows)
|
| 110 |
+
original_q = module.weight[start:stop].clone()
|
| 111 |
+
original_s = module.weight_scale[start:stop].clone()
|
| 112 |
+
original_b = module.bias[start:stop].clone()
|
| 113 |
+
target_t, target_v = ty[:, start:stop], vy[:, start:stop]
|
| 114 |
+
baseline = (integer_projection(vq, vs, original_q) * original_s + original_b.float()).to(torch.bfloat16).float()
|
| 115 |
+
best_loss = (baseline - target_v).square().mean(0)
|
| 116 |
+
before.append(best_loss.clone())
|
| 117 |
+
best_q, best_s, best_b = original_q.clone(), original_s.clone(), original_b.clone()
|
| 118 |
+
labels = torch.zeros(stop - start, device=linear.weight.device, dtype=torch.int32)
|
| 119 |
+
w = rotated[start:stop]
|
| 120 |
+
absmax = w.abs().amax(1).clamp_min(1e-20)
|
| 121 |
+
candidates = [(original_q, original_s)]
|
| 122 |
+
for clip in clips:
|
| 123 |
+
scale = absmax * (clip / 127)
|
| 124 |
+
q = (w / scale[:, None]).round().clamp(-127, 127).to(torch.int8)
|
| 125 |
+
candidates.append((q, scale))
|
| 126 |
+
for candidate_index, (q, prior_s) in enumerate(candidates):
|
| 127 |
+
u_train = integer_projection(tq, ts, q)
|
| 128 |
+
u_val = integer_projection(vq, vs, q)
|
| 129 |
+
for affine in (False, True):
|
| 130 |
+
gain, offset = fit_affine(u_train * prior_s, target_t - original_b.float(),
|
| 131 |
+
affine=affine, ridge=ridge, max_gain=max_gain)
|
| 132 |
+
scale = (prior_s * gain).contiguous()
|
| 133 |
+
bias = (original_b.float() + offset).to(torch.bfloat16)
|
| 134 |
+
predicted = (u_val * scale + bias.float()).to(torch.bfloat16).float()
|
| 135 |
+
loss = (predicted - target_v).square().mean(0)
|
| 136 |
+
better = torch.isfinite(loss) & (loss < best_loss * (1 - min_improvement))
|
| 137 |
+
best_loss = torch.where(better, loss, best_loss)
|
| 138 |
+
best_q = torch.where(better[:, None], q, best_q)
|
| 139 |
+
best_s = torch.where(better, scale, best_s)
|
| 140 |
+
best_b = torch.where(better, bias, best_b)
|
| 141 |
+
labels = torch.where(better, 1 + candidate_index * 2 + int(affine), labels)
|
| 142 |
+
module.weight[start:stop].copy_(best_q)
|
| 143 |
+
module.weight_scale[start:stop].copy_(best_s)
|
| 144 |
+
module.bias[start:stop].copy_(best_b)
|
| 145 |
+
selected.append(labels.cpu())
|
| 146 |
+
after.append(best_loss)
|
| 147 |
+
before, after = torch.cat(before), torch.cat(after)
|
| 148 |
+
labels = torch.cat(selected)
|
| 149 |
+
stats = {"train_rows": tx.shape[0], "selection_rows": vx.shape[0], "output_rows": module.out_features,
|
| 150 |
+
"changed_rows": int((labels != 0).sum()), "clipping": list(clips),
|
| 151 |
+
"selection_mse_before": float(before.mean()), "selection_mse_after": float(after.mean()),
|
| 152 |
+
"selection_mse_ratio": float(torch.where(before.sum() > 0, after.sum() / before.sum().clamp_min(1e-20), 1)),
|
| 153 |
+
"choices": torch.bincount(labels, minlength=1 + len(candidates) * 2).tolist()}
|
| 154 |
+
return module, stats
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
class CalibrationWriter:
|
| 158 |
+
"""Write bounded, tensor-preserving safetensors shards without a full state dict."""
|
| 159 |
+
|
| 160 |
+
def __init__(self, directory, metadata, max_bytes=1_900_000_000):
|
| 161 |
+
if type(max_bytes) is not int or max_bytes < 1:
|
| 162 |
+
raise ValueError("Shard size must be a positive integer")
|
| 163 |
+
self.directory = Path(directory)
|
| 164 |
+
self.directory.mkdir(parents=True, exist_ok=True)
|
| 165 |
+
if any(self.directory.iterdir()):
|
| 166 |
+
raise ValueError("Calibration output directory must be empty")
|
| 167 |
+
self.metadata, self.max_bytes = metadata, max_bytes
|
| 168 |
+
self.pending, self.pending_bytes, self.total_bytes = {}, 0, 0
|
| 169 |
+
self.shards, self.weight_map = [], {}
|
| 170 |
+
self.stem = "clef_text_int8_convrot_calibrated"
|
| 171 |
+
|
| 172 |
+
def add(self, name, module):
|
| 173 |
+
for leaf in ("weight", "weight_scale", "bias"):
|
| 174 |
+
key, tensor = name + "." + leaf, getattr(module, leaf)
|
| 175 |
+
size = tensor.numel() * tensor.element_size()
|
| 176 |
+
if key in self.weight_map or key in self.pending:
|
| 177 |
+
raise ValueError("Duplicate calibrated tensor")
|
| 178 |
+
if self.pending and self.pending_bytes + size > self.max_bytes:
|
| 179 |
+
self.flush()
|
| 180 |
+
self.pending[key] = tensor.detach().cpu().contiguous()
|
| 181 |
+
self.pending_bytes += size
|
| 182 |
+
self.total_bytes += size
|
| 183 |
+
|
| 184 |
+
def flush(self):
|
| 185 |
+
if not self.pending:
|
| 186 |
+
return
|
| 187 |
+
name = f"{self.stem}-{len(self.shards) + 1:05d}.safetensors"
|
| 188 |
+
save_file(self.pending, self.directory / name, metadata={"format": "clef-convrot-linear-overrides-v1"})
|
| 189 |
+
self.weight_map.update({key: name for key in self.pending})
|
| 190 |
+
self.shards.append(name)
|
| 191 |
+
self.pending, self.pending_bytes = {}, 0
|
| 192 |
+
|
| 193 |
+
def finish(self):
|
| 194 |
+
self.flush()
|
| 195 |
+
if not self.shards:
|
| 196 |
+
raise ValueError("No calibrated weights")
|
| 197 |
+
renamed = {}
|
| 198 |
+
for i, old in enumerate(self.shards, 1):
|
| 199 |
+
new = f"{self.stem}-{i:05d}-of-{len(self.shards):05d}.safetensors"
|
| 200 |
+
(self.directory / old).rename(self.directory / new)
|
| 201 |
+
renamed[old] = new
|
| 202 |
+
path = self.directory / (self.stem + ".safetensors.index.json")
|
| 203 |
+
path.write_text(json.dumps({"metadata": {**self.metadata, "total_size": self.total_bytes,
|
| 204 |
+
"format": "clef-convrot-linear-overrides-v1", "group_size": 256},
|
| 205 |
+
"weight_map": {k: renamed[v] for k, v in self.weight_map.items()}}, indent=2) + "\n")
|
| 206 |
+
return path
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def unique_object(pairs):
|
| 210 |
+
result = {}
|
| 211 |
+
for key, value in pairs:
|
| 212 |
+
if key in result:
|
| 213 |
+
raise ValueError("Duplicate calibration index key")
|
| 214 |
+
result[key] = value
|
| 215 |
+
return result
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
class CalibrationIndex:
|
| 219 |
+
def __init__(self, path, text, scope):
|
| 220 |
+
self.path = Path(path)
|
| 221 |
+
index = json.loads(self.path.read_text(), object_pairs_hook=unique_object)
|
| 222 |
+
if (not isinstance(index, dict) or not isinstance(index.get("metadata"), dict)
|
| 223 |
+
or not isinstance(index.get("weight_map"), dict)):
|
| 224 |
+
raise ValueError("Malformed calibration index")
|
| 225 |
+
self.metadata, self.weight_map = index["metadata"], index["weight_map"]
|
| 226 |
+
if (self.metadata.get("format") != "clef-convrot-linear-overrides-v1"
|
| 227 |
+
or self.metadata.get("group_size") != 256 or self.metadata.get("scope") != scope):
|
| 228 |
+
raise ValueError("Calibration format/group/scope mismatch")
|
| 229 |
+
self.names = projection_names(text, scope)
|
| 230 |
+
expected = {name + "." + leaf for name in self.names for leaf in ("weight", "weight_scale", "bias")}
|
| 231 |
+
if set(self.weight_map) != expected:
|
| 232 |
+
raise ValueError("Incomplete or unexpected calibrated tensor coverage")
|
| 233 |
+
if any(not isinstance(filename, str) for filename in self.weight_map.values()):
|
| 234 |
+
raise ValueError("Unsafe calibration shard reference")
|
| 235 |
+
self.files, total = {}, 0
|
| 236 |
+
for filename in sorted(set(self.weight_map.values())):
|
| 237 |
+
if (not isinstance(filename, str) or not filename or "/" in filename or "\\" in filename
|
| 238 |
+
or filename in {".", ".."} or ":" in filename):
|
| 239 |
+
raise ValueError("Unsafe calibration shard reference")
|
| 240 |
+
resolved = (self.path.parent / filename).resolve()
|
| 241 |
+
if not resolved.is_relative_to(self.path.parent.resolve()) or not resolved.is_file():
|
| 242 |
+
raise ValueError("Missing or unsafe calibration shard")
|
| 243 |
+
with safe_open(resolved, framework="pt", device="cpu") as f:
|
| 244 |
+
assigned = {key for key, value in self.weight_map.items() if value == filename}
|
| 245 |
+
if set(f.keys()) != assigned:
|
| 246 |
+
raise ValueError("Calibration shard/index tensor mismatch")
|
| 247 |
+
for key in assigned:
|
| 248 |
+
name, leaf = key.rsplit(".", 1)
|
| 249 |
+
linear = text.get_submodule(name)
|
| 250 |
+
shape, dtype, size = ((linear.out_features, linear.in_features), "I8", 1) if leaf == "weight" else (
|
| 251 |
+
((linear.out_features,), "F32", 4) if leaf == "weight_scale" else ((linear.out_features,), "BF16", 2))
|
| 252 |
+
view = f.get_slice(key)
|
| 253 |
+
if tuple(view.get_shape()) != shape or view.get_dtype() != dtype:
|
| 254 |
+
raise ValueError("Calibrated tensor shape/dtype mismatch")
|
| 255 |
+
total += size * torch.Size(shape).numel()
|
| 256 |
+
self.files[filename] = resolved
|
| 257 |
+
if type(self.metadata.get("total_size")) is not int or total != self.metadata["total_size"]:
|
| 258 |
+
raise ValueError("Calibration total_size mismatch")
|
| 259 |
+
|
| 260 |
+
def load(self, name, linear, *, backend, tile, cluster):
|
| 261 |
+
tensors = []
|
| 262 |
+
for leaf in ("weight", "weight_scale", "bias"):
|
| 263 |
+
key = name + "." + leaf
|
| 264 |
+
with safe_open(self.files[self.weight_map[key]], framework="pt", device=str(linear.weight.device)) as f:
|
| 265 |
+
tensor = f.get_tensor(key)
|
| 266 |
+
if not torch.isfinite(tensor).all() or (leaf == "weight_scale" and (tensor <= 0).any()):
|
| 267 |
+
raise ValueError("Non-finite calibrated tensor or nonpositive scale")
|
| 268 |
+
tensors.append(tensor)
|
| 269 |
+
module = ConvRotLinear.__new__(ConvRotLinear)
|
| 270 |
+
torch.nn.Module.__init__(module)
|
| 271 |
+
module.in_features, module.out_features = linear.in_features, linear.out_features
|
| 272 |
+
module.group_size, module.backend, module.tile, module.cluster = 256, backend, tile, cluster
|
| 273 |
+
for leaf, tensor in zip(("weight", "weight_scale", "bias"), tensors, strict=True):
|
| 274 |
+
module.register_buffer(leaf, tensor)
|
| 275 |
+
return module.eval()
|
runtime/clef_int8_cutlass.py
ADDED
|
@@ -0,0 +1,1307 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
| 2 |
+
# SPDX-License-Identifier: BSD-3-Clause
|
| 3 |
+
|
| 4 |
+
# Redistribution and use in source and binary forms, with or without
|
| 5 |
+
# modification, are permitted provided that the following conditions are met:
|
| 6 |
+
|
| 7 |
+
# 1. Redistributions of source code must retain the above copyright notice, this
|
| 8 |
+
# list of conditions and the following disclaimer.
|
| 9 |
+
|
| 10 |
+
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
| 11 |
+
# this list of conditions and the following disclaimer in the documentation
|
| 12 |
+
# and/or other materials provided with the distribution.
|
| 13 |
+
|
| 14 |
+
# 3. Neither the name of the copyright holder nor the names of its
|
| 15 |
+
# contributors may be used to endorse or promote products derived from
|
| 16 |
+
# this software without specific prior written permission.
|
| 17 |
+
|
| 18 |
+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
| 19 |
+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
| 20 |
+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
| 21 |
+
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
| 22 |
+
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
| 23 |
+
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
| 24 |
+
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
| 25 |
+
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
| 26 |
+
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
| 27 |
+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
| 28 |
+
|
| 29 |
+
from typing import Type
|
| 30 |
+
import math
|
| 31 |
+
import cuda.bindings.driver as cuda
|
| 32 |
+
|
| 33 |
+
import cutlass
|
| 34 |
+
import cutlass.cute as cute
|
| 35 |
+
import cutlass.utils as utils
|
| 36 |
+
import cutlass.pipeline as pipeline
|
| 37 |
+
from cutlass.pipeline import pipeline_init_arrive, pipeline_init_wait
|
| 38 |
+
import cutlass.utils.hopper_helpers as sm90_utils
|
| 39 |
+
from clef_vision_norm import mul_rn
|
| 40 |
+
|
| 41 |
+
# Adapted from CUTLASS 4.7.1 examples/python/CuTeDSL/hopper/dense_gemm.py.
|
| 42 |
+
|
| 43 |
+
"""
|
| 44 |
+
A high-performance batched dense GEMM (C = A * B) example for the NVIDIA Hopper architecture
|
| 45 |
+
using CuTe DSL.
|
| 46 |
+
- Matrix A is MxKxL, L is batch dimension, A can be row-major("K") or column-major("M")
|
| 47 |
+
- Matrix B is NxKxL, L is batch dimension, B can be row-major("N") or column-major("K")
|
| 48 |
+
- Matrix C is MxNxL, L is batch dimension, C can be row-major("N") or column-major("M")
|
| 49 |
+
|
| 50 |
+
This GEMM kernel supports the following features:
|
| 51 |
+
- Utilizes Tensor Memory Access (TMA) for efficient memory operations
|
| 52 |
+
- Utilizes Hopper's WGMMA for matrix multiply-accumulate (MMA) operations
|
| 53 |
+
- Implements TMA multicast with cluster to reduce L2 memory traffic
|
| 54 |
+
- Supports multi-stage pipeline to overlap computation and memory access
|
| 55 |
+
|
| 56 |
+
This GEMM works as follows:
|
| 57 |
+
1. Load A and B matrices from global memory (GMEM) to shared memory (SMEM) using TMA operations.
|
| 58 |
+
2. Perform matrix multiply-accumulate (MMA) operations using WGMMA instruction.
|
| 59 |
+
3. Store results from registers (RMEM) to shared memory (SMEM), then to global memory (GMEM) with TMA operations.
|
| 60 |
+
|
| 61 |
+
Hopper WGMMA instructions operate as follows:
|
| 62 |
+
- Read matrix A from SMEM
|
| 63 |
+
- Read matrix B from SMEM
|
| 64 |
+
- Perform MMA operation and store the result in Accumulator(register)
|
| 65 |
+
|
| 66 |
+
To run this example:
|
| 67 |
+
|
| 68 |
+
.. code-block:: bash
|
| 69 |
+
|
| 70 |
+
python examples/hopper/dense_gemm.py \
|
| 71 |
+
--mnkl 8192,8192,8192,1 --tile_shape_mn 128,256 \
|
| 72 |
+
--cluster_shape_mn 1,1 --a_dtype Float16 --b_dtype Float16 \
|
| 73 |
+
--c_dtype Float16 --acc_dtype Float32 \
|
| 74 |
+
--a_major k --b_major k --c_major n
|
| 75 |
+
|
| 76 |
+
The above example command compute batched gemm with M=8192, N=8192, K=8192,
|
| 77 |
+
batch_count=1. The Hopper WGMMA tile shape is 128x256x64 and the cluster shape
|
| 78 |
+
is (1,1). The input, mma accumulator and output data type are set as fp16, fp32
|
| 79 |
+
and fp16, respectively.
|
| 80 |
+
|
| 81 |
+
To collect performance with NCU profiler:
|
| 82 |
+
|
| 83 |
+
.. code-block:: bash
|
| 84 |
+
|
| 85 |
+
ncu python examples/hopper/dense_gemm.py \
|
| 86 |
+
--mnkl 8192,8192,8192,1 --tile_shape_mn 128,256 \
|
| 87 |
+
--cluster_shape_mn 1,1 --a_dtype Float16 --b_dtype Float16 \
|
| 88 |
+
--c_dtype Float16 --acc_dtype Float32 \
|
| 89 |
+
--a_major k --b_major k --c_major n
|
| 90 |
+
|
| 91 |
+
Constraints:
|
| 92 |
+
* Supported input data types: fp16, fp8 (e4m3fn, e5m2), int8, uint8
|
| 93 |
+
* For fp16 types, A and B must have the same data type
|
| 94 |
+
* For fp8 types, A and B can have different types (e4m3fn or e5m2)
|
| 95 |
+
* For 8-bit integer types, A and B can have different types (int8 or uint8)
|
| 96 |
+
* 8-bit types (e4m3fn, e5m2, int8, uint8) only support k-major layout
|
| 97 |
+
* CTA tile shape M must be 64/128
|
| 98 |
+
* CTA tile shape N must be 64/128/256
|
| 99 |
+
* Cluster shape M/N must be positive and power of 2, total cluster size <= 4
|
| 100 |
+
* The contiguous dimension of A/B/C tensors must be at least 16 bytes aligned,
|
| 101 |
+
i.e, number of elements is a multiple of 8, 16 for Float16, and Float8, respectively.
|
| 102 |
+
"""
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
class HopperWgmmaGemmKernel:
|
| 106 |
+
"""
|
| 107 |
+
This class implements batched matrix multiplication (C = A x B) with support for various data types
|
| 108 |
+
and architectural features specific to Hopper GPUs.
|
| 109 |
+
|
| 110 |
+
:param acc_dtype: Data type for accumulation during computation
|
| 111 |
+
:type acc_dtype: type[cutlass.Numeric]
|
| 112 |
+
:param tile_shape_mn: Shape of the CTA tile (M,N)
|
| 113 |
+
:type tile_shape_mn: Tuple[int, int]
|
| 114 |
+
:param cluster_shape_mn: Cluster dimensions (M,N) for parallel processing
|
| 115 |
+
:type cluster_shape_mn: Tuple[int, int]
|
| 116 |
+
|
| 117 |
+
:note: Supported A/B data types:
|
| 118 |
+
- Float16
|
| 119 |
+
A and B must have the same data type
|
| 120 |
+
- Float8E4M3FN/Float8E5M2
|
| 121 |
+
A and B can have different types (Float8E4M3FN/Float8E5M2)
|
| 122 |
+
only support k-major layout
|
| 123 |
+
- Int8/Uint8
|
| 124 |
+
A and B can have different types (Int8/Uint8)
|
| 125 |
+
only support k-major layout
|
| 126 |
+
|
| 127 |
+
:note: Supported accumulation types:
|
| 128 |
+
- Float32/Float16 (for all floating point inputs)
|
| 129 |
+
- Int32 (for Int8/Uint8 inputs)
|
| 130 |
+
|
| 131 |
+
:note: Constraints:
|
| 132 |
+
- CTA tile M must be 64/128
|
| 133 |
+
- CTA tile N must be 64/128/256
|
| 134 |
+
- CTA tile K must be 64
|
| 135 |
+
- Cluster shape M/N must be positive and power of 2, total cluster size <= 4
|
| 136 |
+
|
| 137 |
+
Example:
|
| 138 |
+
>>> gemm = HopperWgmmaGemmKernel(
|
| 139 |
+
... acc_dtype=cutlass.Float32,
|
| 140 |
+
... tile_shape_mn=(128, 256),
|
| 141 |
+
... cluster_shape_mn=(1, 1)
|
| 142 |
+
... )
|
| 143 |
+
>>> gemm(a_tensor, b_tensor, c_tensor, stream)
|
| 144 |
+
"""
|
| 145 |
+
|
| 146 |
+
def __init__(
|
| 147 |
+
self,
|
| 148 |
+
acc_dtype: type[cutlass.Numeric],
|
| 149 |
+
tile_shape_mn: tuple[int, int],
|
| 150 |
+
cluster_shape_mn: tuple[int, int],
|
| 151 |
+
):
|
| 152 |
+
"""
|
| 153 |
+
Initializes the configuration for a Hopper dense GEMM kernel.
|
| 154 |
+
|
| 155 |
+
This configuration includes data types for operands, tile shape, cluster configuration,
|
| 156 |
+
and thread layout.
|
| 157 |
+
|
| 158 |
+
:param acc_dtype: Data type for accumulation during computation
|
| 159 |
+
:type acc_dtype: type[cutlass.Numeric]
|
| 160 |
+
:param tile_shape_mn: Shape of the CTA tile (M,N)
|
| 161 |
+
:type tile_shape_mn: Tuple[int, int]
|
| 162 |
+
:param cluster_shape_mn: Cluster dimensions (M,N) for parallel processing
|
| 163 |
+
:type cluster_shape_mn: Tuple[int, int]
|
| 164 |
+
"""
|
| 165 |
+
|
| 166 |
+
self.acc_dtype = acc_dtype
|
| 167 |
+
|
| 168 |
+
self.cluster_shape_mn = cluster_shape_mn
|
| 169 |
+
self.mma_inst_shape_mn = None
|
| 170 |
+
# K dimension is deferred in _setup_attributes
|
| 171 |
+
self.tile_shape_mnk = (*tile_shape_mn, 1)
|
| 172 |
+
# For large tile size, using two warp groups is preferred because using only one warp
|
| 173 |
+
# group may result in register spill
|
| 174 |
+
self.atom_layout_mnk = (
|
| 175 |
+
(2, 1, 1)
|
| 176 |
+
if self.tile_shape_mnk[0] > 64
|
| 177 |
+
else (1, 1, 1)
|
| 178 |
+
)
|
| 179 |
+
self.num_mcast_ctas_a = None
|
| 180 |
+
self.num_mcast_ctas_b = None
|
| 181 |
+
self.is_a_mcast = False
|
| 182 |
+
self.is_b_mcast = False
|
| 183 |
+
self.tiled_mma = None
|
| 184 |
+
|
| 185 |
+
self.occupancy = 1
|
| 186 |
+
self.mma_warp_groups = math.prod(self.atom_layout_mnk)
|
| 187 |
+
self.num_threads_per_warp_group = 128
|
| 188 |
+
self.threads_per_cta = self.mma_warp_groups * self.num_threads_per_warp_group
|
| 189 |
+
self.smem_capacity = utils.get_smem_capacity_in_bytes("sm_90")
|
| 190 |
+
|
| 191 |
+
self.ab_stage = None
|
| 192 |
+
self.epi_stage = None
|
| 193 |
+
|
| 194 |
+
self.a_smem_layout_staged = None
|
| 195 |
+
self.b_smem_layout_staged = None
|
| 196 |
+
self.epi_smem_layout_staged = None
|
| 197 |
+
self.epi_tile = None
|
| 198 |
+
|
| 199 |
+
self.shared_storage = None
|
| 200 |
+
self.buffer_align_bytes = 1024
|
| 201 |
+
|
| 202 |
+
def _setup_attributes(self):
|
| 203 |
+
"""Set up configurations that are dependent on GEMM inputs
|
| 204 |
+
|
| 205 |
+
This method configures various attributes based on the input tensor properties
|
| 206 |
+
(data types, leading dimensions) and kernel settings:
|
| 207 |
+
- Configuring tiled MMA
|
| 208 |
+
- Computing MMA/cluster/tile shapes
|
| 209 |
+
- Computing cluster layout
|
| 210 |
+
- Computing multicast CTAs for A/B
|
| 211 |
+
- Computing epilogue subtile
|
| 212 |
+
- Setting up A/B/C stage counts in shared memory
|
| 213 |
+
- Computing A/B/C shared memory layout
|
| 214 |
+
"""
|
| 215 |
+
|
| 216 |
+
# check the cta tile shape
|
| 217 |
+
if self.tile_shape_mnk[0] not in [64, 128]:
|
| 218 |
+
raise ValueError("CTA tile shape M must be 64/128")
|
| 219 |
+
if self.tile_shape_mnk[1] not in [64, 128, 256]:
|
| 220 |
+
raise ValueError("CTA tile shape N must be 64/128/256")
|
| 221 |
+
|
| 222 |
+
self.tiled_mma = sm90_utils.make_trivial_tiled_mma(
|
| 223 |
+
self.a_dtype,
|
| 224 |
+
self.b_dtype,
|
| 225 |
+
self.a_layout.sm90_mma_major_mode(),
|
| 226 |
+
self.b_layout.sm90_mma_major_mode(),
|
| 227 |
+
self.acc_dtype,
|
| 228 |
+
self.atom_layout_mnk,
|
| 229 |
+
tiler_mn=(64, self.tile_shape_mnk[1]),
|
| 230 |
+
)
|
| 231 |
+
mma_inst_shape_k = cute.size(self.tiled_mma.shape_mnk, mode=[2])
|
| 232 |
+
mma_inst_tile_k = 4
|
| 233 |
+
self.tile_shape_mnk = (
|
| 234 |
+
self.tile_shape_mnk[0],
|
| 235 |
+
self.tile_shape_mnk[1],
|
| 236 |
+
mma_inst_shape_k * mma_inst_tile_k,
|
| 237 |
+
)
|
| 238 |
+
|
| 239 |
+
self.cta_layout_mnk = cute.make_layout((*self.cluster_shape_mn, 1))
|
| 240 |
+
self.num_mcast_ctas_a = self.cluster_shape_mn[1]
|
| 241 |
+
self.num_mcast_ctas_b = self.cluster_shape_mn[0]
|
| 242 |
+
self.is_a_mcast = self.num_mcast_ctas_a > 1
|
| 243 |
+
self.is_b_mcast = self.num_mcast_ctas_b > 1
|
| 244 |
+
|
| 245 |
+
is_cooperative = self.atom_layout_mnk == (2, 1, 1)
|
| 246 |
+
self.epi_tile = sm90_utils.compute_tile_shape_or_override(
|
| 247 |
+
self.tile_shape_mnk, self.c_dtype, is_cooperative=is_cooperative
|
| 248 |
+
)
|
| 249 |
+
|
| 250 |
+
# Compute stage before compute smem layout
|
| 251 |
+
self.ab_stage, self.epi_stage = self._compute_stages(
|
| 252 |
+
self.tile_shape_mnk,
|
| 253 |
+
self.a_dtype,
|
| 254 |
+
self.b_dtype,
|
| 255 |
+
self.smem_capacity,
|
| 256 |
+
self.occupancy,
|
| 257 |
+
)
|
| 258 |
+
|
| 259 |
+
(
|
| 260 |
+
self.a_smem_layout_staged,
|
| 261 |
+
self.b_smem_layout_staged,
|
| 262 |
+
self.epi_smem_layout_staged,
|
| 263 |
+
) = self._make_smem_layouts(
|
| 264 |
+
self.tile_shape_mnk,
|
| 265 |
+
self.epi_tile,
|
| 266 |
+
self.a_dtype,
|
| 267 |
+
self.a_layout,
|
| 268 |
+
self.b_dtype,
|
| 269 |
+
self.b_layout,
|
| 270 |
+
self.ab_stage,
|
| 271 |
+
self.c_dtype,
|
| 272 |
+
self.c_layout,
|
| 273 |
+
self.epi_stage,
|
| 274 |
+
)
|
| 275 |
+
|
| 276 |
+
@cute.jit
|
| 277 |
+
def __call__(
|
| 278 |
+
self,
|
| 279 |
+
a: cute.Tensor,
|
| 280 |
+
b: cute.Tensor,
|
| 281 |
+
c: cute.Tensor,
|
| 282 |
+
scale_a: cute.Tensor,
|
| 283 |
+
scale_b: cute.Tensor,
|
| 284 |
+
bias: cute.Tensor,
|
| 285 |
+
stream: cuda.CUstream,
|
| 286 |
+
):
|
| 287 |
+
"""Execute the GEMM operation in steps:
|
| 288 |
+
- Setup static attributes
|
| 289 |
+
- Setup TMA load/store atoms and tensors
|
| 290 |
+
- Compute grid size
|
| 291 |
+
- Define shared storage for kernel
|
| 292 |
+
- Launch the kernel synchronously
|
| 293 |
+
|
| 294 |
+
:param a: Input tensor A
|
| 295 |
+
:type a: cute.Tensor
|
| 296 |
+
:param b: Input tensor B
|
| 297 |
+
:type b: cute.Tensor
|
| 298 |
+
:param c: Output tensor C
|
| 299 |
+
:type c: cute.Tensor
|
| 300 |
+
:param stream: CUDA stream for asynchronous execution
|
| 301 |
+
:type stream: cuda.CUstream
|
| 302 |
+
"""
|
| 303 |
+
|
| 304 |
+
# setup static attributes before smem/grid/tma computation
|
| 305 |
+
self.a_dtype = a.element_type
|
| 306 |
+
self.b_dtype = b.element_type
|
| 307 |
+
self.c_dtype = c.element_type
|
| 308 |
+
self.a_layout = utils.LayoutEnum.from_tensor(a)
|
| 309 |
+
self.b_layout = utils.LayoutEnum.from_tensor(b)
|
| 310 |
+
self.c_layout = utils.LayoutEnum.from_tensor(c)
|
| 311 |
+
|
| 312 |
+
if cutlass.const_expr(
|
| 313 |
+
self.a_dtype.width == 16 and self.a_dtype != self.b_dtype
|
| 314 |
+
):
|
| 315 |
+
raise TypeError(f"Type mismatch: {self.a_dtype} != {self.b_dtype}")
|
| 316 |
+
if cutlass.const_expr(self.a_dtype.width != self.b_dtype.width):
|
| 317 |
+
raise TypeError(
|
| 318 |
+
f"Type width mismatch: {self.a_dtype.width} != {self.b_dtype.width}"
|
| 319 |
+
)
|
| 320 |
+
if cutlass.const_expr(self.a_dtype.width != 16 and self.a_dtype.width != 8):
|
| 321 |
+
raise TypeError("a_dtype should be float16 or float8")
|
| 322 |
+
|
| 323 |
+
self._setup_attributes()
|
| 324 |
+
|
| 325 |
+
tma_atom_a, tma_tensor_a = self._make_tma_atoms_and_tensors(
|
| 326 |
+
a,
|
| 327 |
+
self.a_smem_layout_staged,
|
| 328 |
+
(self.tile_shape_mnk[0], self.tile_shape_mnk[2]),
|
| 329 |
+
self.cluster_shape_mn[1],
|
| 330 |
+
)
|
| 331 |
+
|
| 332 |
+
tma_atom_b, tma_tensor_b = self._make_tma_atoms_and_tensors(
|
| 333 |
+
b,
|
| 334 |
+
self.b_smem_layout_staged,
|
| 335 |
+
(self.tile_shape_mnk[1], self.tile_shape_mnk[2]),
|
| 336 |
+
self.cluster_shape_mn[0],
|
| 337 |
+
)
|
| 338 |
+
|
| 339 |
+
tma_atom_c, tma_tensor_c = self._make_tma_store_atoms_and_tensors(
|
| 340 |
+
c,
|
| 341 |
+
self.epi_smem_layout_staged,
|
| 342 |
+
self.epi_tile,
|
| 343 |
+
)
|
| 344 |
+
|
| 345 |
+
grid = self._compute_grid(c, self.tile_shape_mnk, self.cluster_shape_mn)
|
| 346 |
+
|
| 347 |
+
@cute.struct
|
| 348 |
+
class SharedStorage:
|
| 349 |
+
mainloop_pipeline_array_ptr: cute.struct.MemRange[
|
| 350 |
+
cutlass.Int64, self.ab_stage * 2
|
| 351 |
+
]
|
| 352 |
+
sA: cute.struct.Align[
|
| 353 |
+
cute.struct.MemRange[
|
| 354 |
+
self.a_dtype, cute.cosize(self.a_smem_layout_staged)
|
| 355 |
+
],
|
| 356 |
+
self.buffer_align_bytes,
|
| 357 |
+
]
|
| 358 |
+
sB: cute.struct.Align[
|
| 359 |
+
cute.struct.MemRange[
|
| 360 |
+
self.b_dtype, cute.cosize(self.b_smem_layout_staged)
|
| 361 |
+
],
|
| 362 |
+
self.buffer_align_bytes,
|
| 363 |
+
]
|
| 364 |
+
|
| 365 |
+
self.shared_storage = SharedStorage
|
| 366 |
+
|
| 367 |
+
# Launch the kernel synchronously
|
| 368 |
+
self.kernel(
|
| 369 |
+
tma_atom_a,
|
| 370 |
+
tma_tensor_a,
|
| 371 |
+
tma_atom_b,
|
| 372 |
+
tma_tensor_b,
|
| 373 |
+
tma_atom_c,
|
| 374 |
+
tma_tensor_c,
|
| 375 |
+
scale_a,
|
| 376 |
+
scale_b,
|
| 377 |
+
bias,
|
| 378 |
+
self.tiled_mma,
|
| 379 |
+
self.cta_layout_mnk,
|
| 380 |
+
self.a_smem_layout_staged,
|
| 381 |
+
self.b_smem_layout_staged,
|
| 382 |
+
self.epi_smem_layout_staged,
|
| 383 |
+
).launch(
|
| 384 |
+
grid=grid,
|
| 385 |
+
block=[self.threads_per_cta, 1, 1],
|
| 386 |
+
cluster=(*self.cluster_shape_mn, 1),
|
| 387 |
+
stream=stream,
|
| 388 |
+
)
|
| 389 |
+
return
|
| 390 |
+
|
| 391 |
+
# GPU device kernel
|
| 392 |
+
@cute.kernel
|
| 393 |
+
def kernel(
|
| 394 |
+
self,
|
| 395 |
+
tma_atom_a: cute.CopyAtom,
|
| 396 |
+
mA_mkl: cute.Tensor,
|
| 397 |
+
tma_atom_b: cute.CopyAtom,
|
| 398 |
+
mB_nkl: cute.Tensor,
|
| 399 |
+
tma_atom_c: cute.CopyAtom,
|
| 400 |
+
mC_mnl: cute.Tensor,
|
| 401 |
+
mScaleA: cute.Tensor,
|
| 402 |
+
mScaleB: cute.Tensor,
|
| 403 |
+
mBias: cute.Tensor,
|
| 404 |
+
tiled_mma: cute.TiledMma,
|
| 405 |
+
cta_layout_mnk: cute.Layout,
|
| 406 |
+
a_smem_layout_staged: cute.ComposedLayout,
|
| 407 |
+
b_smem_layout_staged: cute.ComposedLayout,
|
| 408 |
+
epi_smem_layout_staged: cute.ComposedLayout,
|
| 409 |
+
):
|
| 410 |
+
"""
|
| 411 |
+
GPU device kernel performing the batched GEMM computation.
|
| 412 |
+
|
| 413 |
+
:param tma_atom_a: TMA copy atom for A tensor
|
| 414 |
+
:type tma_atom_a: cute.CopyAtom
|
| 415 |
+
:param mA_mkl: Input tensor A
|
| 416 |
+
:type mA_mkl: cute.Tensor
|
| 417 |
+
:param tma_atom_b: TMA copy atom for B tensor
|
| 418 |
+
:type tma_atom_b: cute.CopyAtom
|
| 419 |
+
:param mB_nkl: Input tensor B
|
| 420 |
+
:type mB_nkl: cute.Tensor
|
| 421 |
+
:param tma_atom_c: TMA copy atom for C tensor
|
| 422 |
+
:type tma_atom_c: cute.CopyAtom
|
| 423 |
+
:param mC_mnl: Output tensor C
|
| 424 |
+
:type mC_mnl: cute.Tensor
|
| 425 |
+
:param tiled_mma: Tiled MMA object
|
| 426 |
+
:type tiled_mma: cute.TiledMma
|
| 427 |
+
:param cta_layout_mnk: CTA layout
|
| 428 |
+
:type cta_layout_mnk: cute.Layout
|
| 429 |
+
:param a_smem_layout_staged: Shared memory layout for A
|
| 430 |
+
:type a_smem_layout_staged: cute.ComposedLayout
|
| 431 |
+
:param b_smem_layout_staged: Shared memory layout for B
|
| 432 |
+
:type b_smem_layout_staged: cute.ComposedLayout
|
| 433 |
+
:param epi_smem_layout_staged: Shared memory layout for epilogue
|
| 434 |
+
:type epi_smem_layout_staged: cute.ComposedLayout
|
| 435 |
+
"""
|
| 436 |
+
|
| 437 |
+
warp_idx = cute.arch.warp_idx()
|
| 438 |
+
warp_idx = cute.arch.make_warp_uniform(warp_idx)
|
| 439 |
+
|
| 440 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 441 |
+
# Prefetch Tma desc
|
| 442 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 443 |
+
if warp_idx == 0:
|
| 444 |
+
cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_a)
|
| 445 |
+
cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_b)
|
| 446 |
+
|
| 447 |
+
# ///////////////////////////////////////////////////////////////////////////////
|
| 448 |
+
# Get cta/warp/thread idx
|
| 449 |
+
# ///////////////////////////////////////////////////////////////////////////////
|
| 450 |
+
bidx, bidy, bidz = cute.arch.block_idx()
|
| 451 |
+
tidx, _, _ = cute.arch.thread_idx()
|
| 452 |
+
|
| 453 |
+
cidx, cidy, _ = cute.arch.cluster_idx()
|
| 454 |
+
cdimx, cdimy, _ = cute.arch.cluster_dim()
|
| 455 |
+
cluster_id = cidx + cdimx * cidy
|
| 456 |
+
|
| 457 |
+
# CTA Swizzle to promote L2 data reuse
|
| 458 |
+
group_size_m = 8
|
| 459 |
+
s_shape = (
|
| 460 |
+
(group_size_m, cdimx // group_size_m),
|
| 461 |
+
cdimy,
|
| 462 |
+
)
|
| 463 |
+
s_stride = ((1, cdimy * group_size_m), group_size_m)
|
| 464 |
+
s_layout = cute.make_layout(s_shape, stride=s_stride)
|
| 465 |
+
num_reg_cids = cute.size(s_shape)
|
| 466 |
+
cid_m, cid_n = s_layout.get_flat_coord(cluster_id % num_reg_cids)
|
| 467 |
+
|
| 468 |
+
# Deal with the tail part
|
| 469 |
+
if cluster_id >= num_reg_cids:
|
| 470 |
+
tail_size_m = cdimx % group_size_m
|
| 471 |
+
tail_layout = cute.make_layout(
|
| 472 |
+
(tail_size_m, cdimy), stride=(1, tail_size_m)
|
| 473 |
+
)
|
| 474 |
+
tail_cid = cluster_id - num_reg_cids
|
| 475 |
+
tail_cid_m, tail_cid_n = tail_layout.get_flat_coord(tail_cid)
|
| 476 |
+
cid_m = cute.size(s_shape, mode=[0]) + tail_cid_m
|
| 477 |
+
cid_n = tail_cid_n
|
| 478 |
+
|
| 479 |
+
# Get the pid from cluster id
|
| 480 |
+
bidx_in_cluster = cute.arch.block_in_cluster_idx()
|
| 481 |
+
pid_m = cid_m * self.cluster_shape_mn[0] + bidx_in_cluster[0]
|
| 482 |
+
pid_n = cid_n * self.cluster_shape_mn[1] + bidx_in_cluster[1]
|
| 483 |
+
|
| 484 |
+
tile_coord_mnkl = (pid_m, pid_n, None, bidz)
|
| 485 |
+
cta_rank_in_cluster = cute.arch.make_warp_uniform(
|
| 486 |
+
cute.arch.block_idx_in_cluster()
|
| 487 |
+
)
|
| 488 |
+
cluster_coord_mnk = cta_layout_mnk.get_flat_coord(cta_rank_in_cluster)
|
| 489 |
+
|
| 490 |
+
# ///////////////////////////////////////////////////////////////////////////////
|
| 491 |
+
# Get mcast mask
|
| 492 |
+
# ///////////////////////////////////////////////////////////////////////////////
|
| 493 |
+
a_mcast_mask = cute.make_layout_image_mask(
|
| 494 |
+
cta_layout_mnk, cluster_coord_mnk, mode=1
|
| 495 |
+
)
|
| 496 |
+
b_mcast_mask = cute.make_layout_image_mask(
|
| 497 |
+
cta_layout_mnk, cluster_coord_mnk, mode=0
|
| 498 |
+
)
|
| 499 |
+
|
| 500 |
+
a_mcast_mask = a_mcast_mask if self.is_a_mcast else 0
|
| 501 |
+
b_mcast_mask = b_mcast_mask if self.is_b_mcast else 0
|
| 502 |
+
a_smem_layout = cute.slice_(a_smem_layout_staged, (None, None, 0))
|
| 503 |
+
b_smem_layout = cute.slice_(b_smem_layout_staged, (None, None, 0))
|
| 504 |
+
tma_copy_bytes = cute.size_in_bytes(
|
| 505 |
+
self.a_dtype, a_smem_layout
|
| 506 |
+
) + cute.size_in_bytes(self.b_dtype, b_smem_layout)
|
| 507 |
+
|
| 508 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 509 |
+
# Alloc and init AB full/empty + ACC full mbar (pipeline)
|
| 510 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 511 |
+
smem = cutlass.utils.SmemAllocator()
|
| 512 |
+
storage = smem.allocate(self.shared_storage)
|
| 513 |
+
|
| 514 |
+
# mbar arrays
|
| 515 |
+
mainloop_pipeline_array_ptr = storage.mainloop_pipeline_array_ptr.data_ptr()
|
| 516 |
+
|
| 517 |
+
# Threads/warps participating in this pipeline
|
| 518 |
+
mainloop_pipeline_producer_group = pipeline.CooperativeGroup(
|
| 519 |
+
pipeline.Agent.Thread
|
| 520 |
+
)
|
| 521 |
+
# Each warp will constribute to the arrive count with the number of mcast size
|
| 522 |
+
mcast_size = self.num_mcast_ctas_a + self.num_mcast_ctas_b - 1
|
| 523 |
+
num_warps = self.threads_per_cta // 32
|
| 524 |
+
consumer_arrive_cnt = mcast_size * num_warps
|
| 525 |
+
mainloop_pipeline_consumer_group = pipeline.CooperativeGroup(
|
| 526 |
+
pipeline.Agent.Thread, consumer_arrive_cnt
|
| 527 |
+
)
|
| 528 |
+
|
| 529 |
+
cta_layout_vmnk = cute.make_layout((1, *cta_layout_mnk.shape))
|
| 530 |
+
mainloop_pipeline = pipeline.PipelineTmaAsync.create(
|
| 531 |
+
barrier_storage=mainloop_pipeline_array_ptr,
|
| 532 |
+
num_stages=self.ab_stage,
|
| 533 |
+
producer_group=mainloop_pipeline_producer_group,
|
| 534 |
+
consumer_group=mainloop_pipeline_consumer_group,
|
| 535 |
+
tx_count=tma_copy_bytes,
|
| 536 |
+
cta_layout_vmnk=cta_layout_vmnk,
|
| 537 |
+
defer_sync=True,
|
| 538 |
+
)
|
| 539 |
+
|
| 540 |
+
# Cluster arrive after barrier init
|
| 541 |
+
pipeline_init_arrive(cluster_shape_mn=self.cluster_shape_mn, is_relaxed=True)
|
| 542 |
+
|
| 543 |
+
# ///////////////////////////////////////////////////////////////////////////////
|
| 544 |
+
# Generate smem tensor A/B
|
| 545 |
+
# ///////////////////////////////////////////////////////////////////////////////
|
| 546 |
+
sA = storage.sA.get_tensor(
|
| 547 |
+
a_smem_layout_staged.outer, swizzle=a_smem_layout_staged.inner
|
| 548 |
+
)
|
| 549 |
+
sB = storage.sB.get_tensor(
|
| 550 |
+
b_smem_layout_staged.outer, swizzle=b_smem_layout_staged.inner
|
| 551 |
+
)
|
| 552 |
+
sC_ptr = cute.recast_ptr(
|
| 553 |
+
sA.iterator, epi_smem_layout_staged.inner, dtype=self.c_dtype
|
| 554 |
+
)
|
| 555 |
+
sC = cute.make_tensor(sC_ptr, epi_smem_layout_staged.outer)
|
| 556 |
+
|
| 557 |
+
# ///////////////////////////////////////////////////////////////////////////////
|
| 558 |
+
# Local_tile partition global tensors
|
| 559 |
+
# ///////////////////////////////////////////////////////////////////////////////
|
| 560 |
+
# (bM, bK, RestK)
|
| 561 |
+
gA_mkl = cute.local_tile(
|
| 562 |
+
mA_mkl, self.tile_shape_mnk, tile_coord_mnkl, proj=(1, None, 1)
|
| 563 |
+
)
|
| 564 |
+
# (bN, bK, RestK)
|
| 565 |
+
gB_nkl = cute.local_tile(
|
| 566 |
+
mB_nkl, self.tile_shape_mnk, tile_coord_mnkl, proj=(None, 1, 1)
|
| 567 |
+
)
|
| 568 |
+
# (bM, bN)
|
| 569 |
+
gC_mnl = cute.local_tile(
|
| 570 |
+
mC_mnl, self.tile_shape_mnk, tile_coord_mnkl, proj=(1, 1, None)
|
| 571 |
+
)
|
| 572 |
+
cC_mnl = cute.make_identity_tensor(mC_mnl.shape)
|
| 573 |
+
gC_coord = cute.local_tile(
|
| 574 |
+
cC_mnl, self.tile_shape_mnk, tile_coord_mnkl, proj=(1, 1, None)
|
| 575 |
+
)
|
| 576 |
+
|
| 577 |
+
# //////////////////////////////////////////////////////////////////////////////
|
| 578 |
+
# Partition global tensor for TiledMMA_A/B/C
|
| 579 |
+
# //////////////////////////////////////////////////////////////////////////////
|
| 580 |
+
warp_group_idx = cute.arch.make_warp_uniform(
|
| 581 |
+
tidx // self.num_threads_per_warp_group
|
| 582 |
+
)
|
| 583 |
+
warp_group_thread_layout = cute.make_layout(
|
| 584 |
+
self.mma_warp_groups, stride=self.num_threads_per_warp_group
|
| 585 |
+
)
|
| 586 |
+
thr_mma = tiled_mma.get_slice(warp_group_thread_layout(warp_group_idx))
|
| 587 |
+
|
| 588 |
+
tCgC = thr_mma.partition_C(gC_mnl)
|
| 589 |
+
|
| 590 |
+
# //////////////////////////////////////////////////////////////////////////////
|
| 591 |
+
# Partition shared tensor for TMA load A/B
|
| 592 |
+
# //////////////////////////////////////////////////////////////////////////////
|
| 593 |
+
# TMA load A partition_S/D
|
| 594 |
+
a_cta_layout = cute.make_layout(cute.slice_(cta_layout_mnk, (0, None, 0)).shape)
|
| 595 |
+
a_cta_crd = cluster_coord_mnk[1]
|
| 596 |
+
sA_for_tma_partition = cute.group_modes(sA, 0, 2)
|
| 597 |
+
gA_for_tma_partition = cute.group_modes(gA_mkl, 0, 2)
|
| 598 |
+
tAsA, tAgA_mkl = cute.nvgpu.cpasync.tma_partition(
|
| 599 |
+
tma_atom_a,
|
| 600 |
+
a_cta_crd,
|
| 601 |
+
a_cta_layout,
|
| 602 |
+
sA_for_tma_partition,
|
| 603 |
+
gA_for_tma_partition,
|
| 604 |
+
)
|
| 605 |
+
|
| 606 |
+
# TMA load B partition_S/D
|
| 607 |
+
b_cta_layout = cute.make_layout(cute.slice_(cta_layout_mnk, (None, 0, 0)).shape)
|
| 608 |
+
b_cta_crd = cluster_coord_mnk[0]
|
| 609 |
+
sB_for_tma_partition = cute.group_modes(sB, 0, 2)
|
| 610 |
+
gB_for_tma_partition = cute.group_modes(gB_nkl, 0, 2)
|
| 611 |
+
tBsB, tBgB_nkl = cute.nvgpu.cpasync.tma_partition(
|
| 612 |
+
tma_atom_b,
|
| 613 |
+
b_cta_crd,
|
| 614 |
+
b_cta_layout,
|
| 615 |
+
sB_for_tma_partition,
|
| 616 |
+
gB_for_tma_partition,
|
| 617 |
+
)
|
| 618 |
+
|
| 619 |
+
# //////////////////////////////////////////////////////////////////////////////
|
| 620 |
+
# Make fragments
|
| 621 |
+
# //////////////////////////////////////////////////////////////////////////////
|
| 622 |
+
tCsA = thr_mma.partition_A(sA)
|
| 623 |
+
tCsB = thr_mma.partition_B(sB)
|
| 624 |
+
tCrA = tiled_mma.make_fragment_A(tCsA)
|
| 625 |
+
tCrB = tiled_mma.make_fragment_B(tCsB)
|
| 626 |
+
|
| 627 |
+
acc_shape = tCgC.shape
|
| 628 |
+
accumulators = cute.make_rmem_tensor(acc_shape, self.acc_dtype)
|
| 629 |
+
|
| 630 |
+
# ///////////////////////////////////////////////////////////////////////////////
|
| 631 |
+
# Cluster wait
|
| 632 |
+
# ///////////////////////////////////////////////////////////////////////////////
|
| 633 |
+
# cluster wait for barrier init
|
| 634 |
+
pipeline_init_wait(cluster_shape_mn=self.cluster_shape_mn)
|
| 635 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 636 |
+
# Prefetch
|
| 637 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 638 |
+
k_tile_cnt = cute.size(gA_mkl, mode=[2])
|
| 639 |
+
prefetch_k_tile_cnt = cutlass.max(cutlass.min(self.ab_stage, k_tile_cnt), 0)
|
| 640 |
+
|
| 641 |
+
mainloop_producer_state = pipeline.make_pipeline_state(
|
| 642 |
+
pipeline.PipelineUserType.Producer, self.ab_stage
|
| 643 |
+
)
|
| 644 |
+
if warp_idx == 0:
|
| 645 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 646 |
+
# Prefetch TMA load
|
| 647 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 648 |
+
for prefetch_idx in cutlass.range(prefetch_k_tile_cnt, unroll=1):
|
| 649 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 650 |
+
# Wait for A/B buffers to be empty before loading into them
|
| 651 |
+
# Also sets the transaction barrier for the A/B buffers
|
| 652 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 653 |
+
mainloop_pipeline.producer_acquire(mainloop_producer_state)
|
| 654 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 655 |
+
# Slice to global/shared memref to current k_tile
|
| 656 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 657 |
+
tAgA_k = tAgA_mkl[(None, mainloop_producer_state.count)]
|
| 658 |
+
tAsA_pipe = tAsA[(None, mainloop_producer_state.index)]
|
| 659 |
+
|
| 660 |
+
tBgB_k = tBgB_nkl[(None, mainloop_producer_state.count)]
|
| 661 |
+
tBsB_pipe = tBsB[(None, mainloop_producer_state.index)]
|
| 662 |
+
|
| 663 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 664 |
+
# TMA load A/B
|
| 665 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 666 |
+
cute.copy(
|
| 667 |
+
tma_atom_a,
|
| 668 |
+
tAgA_k,
|
| 669 |
+
tAsA_pipe,
|
| 670 |
+
tma_bar_ptr=mainloop_pipeline.producer_get_barrier(
|
| 671 |
+
mainloop_producer_state
|
| 672 |
+
),
|
| 673 |
+
mcast_mask=a_mcast_mask,
|
| 674 |
+
)
|
| 675 |
+
cute.copy(
|
| 676 |
+
tma_atom_b,
|
| 677 |
+
tBgB_k,
|
| 678 |
+
tBsB_pipe,
|
| 679 |
+
tma_bar_ptr=mainloop_pipeline.producer_get_barrier(
|
| 680 |
+
mainloop_producer_state
|
| 681 |
+
),
|
| 682 |
+
mcast_mask=b_mcast_mask,
|
| 683 |
+
)
|
| 684 |
+
# Mainloop pipeline's producer commit is a NOP
|
| 685 |
+
mainloop_pipeline.producer_commit(mainloop_producer_state)
|
| 686 |
+
mainloop_producer_state.advance()
|
| 687 |
+
|
| 688 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 689 |
+
# Prologue MMAs
|
| 690 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 691 |
+
k_pipe_mmas = 1
|
| 692 |
+
|
| 693 |
+
mainloop_consumer_read_state = pipeline.make_pipeline_state(
|
| 694 |
+
pipeline.PipelineUserType.Consumer, self.ab_stage
|
| 695 |
+
)
|
| 696 |
+
mainloop_consumer_release_state = pipeline.make_pipeline_state(
|
| 697 |
+
pipeline.PipelineUserType.Consumer, self.ab_stage
|
| 698 |
+
)
|
| 699 |
+
|
| 700 |
+
peek_ab_full_status = cutlass.Boolean(1)
|
| 701 |
+
if mainloop_consumer_read_state.count < k_tile_cnt:
|
| 702 |
+
peek_ab_full_status = mainloop_pipeline.consumer_try_wait(
|
| 703 |
+
mainloop_consumer_read_state
|
| 704 |
+
)
|
| 705 |
+
|
| 706 |
+
tiled_mma.set(cute.nvgpu.warpgroup.Field.ACCUMULATE, False)
|
| 707 |
+
num_k_blocks = cute.size(tCrA, mode=[2])
|
| 708 |
+
for k_tile in cutlass.range_constexpr(k_pipe_mmas):
|
| 709 |
+
# Wait for A/B buffer to be ready
|
| 710 |
+
mainloop_pipeline.consumer_wait(
|
| 711 |
+
mainloop_consumer_read_state, peek_ab_full_status
|
| 712 |
+
)
|
| 713 |
+
|
| 714 |
+
cute.nvgpu.warpgroup.fence()
|
| 715 |
+
for k_block_idx in cutlass.range(num_k_blocks, unroll_full=True):
|
| 716 |
+
k_block_coord = (
|
| 717 |
+
None,
|
| 718 |
+
None,
|
| 719 |
+
k_block_idx,
|
| 720 |
+
mainloop_consumer_read_state.index,
|
| 721 |
+
)
|
| 722 |
+
tCrA_1phase = tCrA[k_block_coord]
|
| 723 |
+
tCrB_1phase = tCrB[k_block_coord]
|
| 724 |
+
|
| 725 |
+
cute.gemm(
|
| 726 |
+
tiled_mma,
|
| 727 |
+
accumulators,
|
| 728 |
+
tCrA_1phase,
|
| 729 |
+
tCrB_1phase,
|
| 730 |
+
accumulators,
|
| 731 |
+
)
|
| 732 |
+
tiled_mma.set(cute.nvgpu.warpgroup.Field.ACCUMULATE, True)
|
| 733 |
+
|
| 734 |
+
cute.nvgpu.warpgroup.commit_group()
|
| 735 |
+
mainloop_consumer_read_state.advance()
|
| 736 |
+
peek_ab_full_status = cutlass.Boolean(1)
|
| 737 |
+
if mainloop_consumer_read_state.count < k_tile_cnt:
|
| 738 |
+
peek_ab_full_status = mainloop_pipeline.consumer_try_wait(
|
| 739 |
+
mainloop_consumer_read_state
|
| 740 |
+
)
|
| 741 |
+
|
| 742 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 743 |
+
# MAINLOOP
|
| 744 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 745 |
+
for k_tile in cutlass.range(k_pipe_mmas, k_tile_cnt, 1, unroll=1):
|
| 746 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 747 |
+
# Wait for TMA copies to complete
|
| 748 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 749 |
+
mainloop_pipeline.consumer_wait(
|
| 750 |
+
mainloop_consumer_read_state, peek_ab_full_status
|
| 751 |
+
)
|
| 752 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 753 |
+
# WGMMA
|
| 754 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 755 |
+
cute.nvgpu.warpgroup.fence()
|
| 756 |
+
for k_block_idx in cutlass.range(num_k_blocks, unroll_full=True):
|
| 757 |
+
k_block_coord = (
|
| 758 |
+
None,
|
| 759 |
+
None,
|
| 760 |
+
k_block_idx,
|
| 761 |
+
mainloop_consumer_read_state.index,
|
| 762 |
+
)
|
| 763 |
+
tCrA_1phase = tCrA[k_block_coord]
|
| 764 |
+
tCrB_1phase = tCrB[k_block_coord]
|
| 765 |
+
|
| 766 |
+
cute.gemm(
|
| 767 |
+
tiled_mma,
|
| 768 |
+
accumulators,
|
| 769 |
+
tCrA_1phase,
|
| 770 |
+
tCrB_1phase,
|
| 771 |
+
accumulators,
|
| 772 |
+
)
|
| 773 |
+
|
| 774 |
+
cute.nvgpu.warpgroup.commit_group()
|
| 775 |
+
# Wait on the wgmma barrier for previous k_pipe_mmas wgmmas to complete
|
| 776 |
+
cute.nvgpu.warpgroup.wait_group(k_pipe_mmas)
|
| 777 |
+
|
| 778 |
+
mainloop_pipeline.consumer_release(mainloop_consumer_release_state)
|
| 779 |
+
|
| 780 |
+
mainloop_consumer_read_state.advance()
|
| 781 |
+
mainloop_consumer_release_state.advance()
|
| 782 |
+
|
| 783 |
+
peek_ab_full_status = cutlass.Boolean(1)
|
| 784 |
+
if mainloop_consumer_read_state.count < k_tile_cnt:
|
| 785 |
+
peek_ab_full_status = mainloop_pipeline.consumer_try_wait(
|
| 786 |
+
mainloop_consumer_read_state
|
| 787 |
+
)
|
| 788 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 789 |
+
# TMA load
|
| 790 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 791 |
+
if warp_idx == 0 and mainloop_producer_state.count < k_tile_cnt:
|
| 792 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 793 |
+
# Wait for A/B buffers to be empty before loading into them
|
| 794 |
+
# Also sets the transaction barrier for the A/B buffers
|
| 795 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 796 |
+
mainloop_pipeline.producer_acquire(mainloop_producer_state)
|
| 797 |
+
|
| 798 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 799 |
+
# Slice to global/shared memref to current k_tile
|
| 800 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 801 |
+
tAgA_k = tAgA_mkl[(None, mainloop_producer_state.count)]
|
| 802 |
+
tAsA_pipe = tAsA[(None, mainloop_producer_state.index)]
|
| 803 |
+
|
| 804 |
+
tBgB_k = tBgB_nkl[(None, mainloop_producer_state.count)]
|
| 805 |
+
tBsB_pipe = tBsB[(None, mainloop_producer_state.index)]
|
| 806 |
+
|
| 807 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 808 |
+
# TMA load A/B
|
| 809 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 810 |
+
cute.copy(
|
| 811 |
+
tma_atom_a,
|
| 812 |
+
tAgA_k,
|
| 813 |
+
tAsA_pipe,
|
| 814 |
+
tma_bar_ptr=mainloop_pipeline.producer_get_barrier(
|
| 815 |
+
mainloop_producer_state
|
| 816 |
+
),
|
| 817 |
+
mcast_mask=a_mcast_mask,
|
| 818 |
+
)
|
| 819 |
+
cute.copy(
|
| 820 |
+
tma_atom_b,
|
| 821 |
+
tBgB_k,
|
| 822 |
+
tBsB_pipe,
|
| 823 |
+
tma_bar_ptr=mainloop_pipeline.producer_get_barrier(
|
| 824 |
+
mainloop_producer_state
|
| 825 |
+
),
|
| 826 |
+
mcast_mask=b_mcast_mask,
|
| 827 |
+
)
|
| 828 |
+
# Mainloop pipeline's producer commit is a NOP
|
| 829 |
+
mainloop_pipeline.producer_commit(mainloop_producer_state)
|
| 830 |
+
mainloop_producer_state.advance()
|
| 831 |
+
|
| 832 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 833 |
+
# EPILOG
|
| 834 |
+
# /////////////////////////////////////////////////////////////////////////////
|
| 835 |
+
cute.nvgpu.warpgroup.wait_group(0)
|
| 836 |
+
|
| 837 |
+
if cute.size(self.cluster_shape_mn) > 1:
|
| 838 |
+
# Wait for all threads in the cluster to finish, avoid early release of smem
|
| 839 |
+
cute.arch.cluster_arrive()
|
| 840 |
+
cute.arch.cluster_wait()
|
| 841 |
+
else:
|
| 842 |
+
# For cluster that has a single thread block, it might have more than one warp groups.
|
| 843 |
+
# Wait for all warp groups in the thread block to finish, because smem for tensor A in
|
| 844 |
+
# the mainloop is reused in the epilogue.
|
| 845 |
+
cute.arch.sync_threads()
|
| 846 |
+
|
| 847 |
+
copy_atom_r2s = sm90_utils.sm90_get_smem_store_op(
|
| 848 |
+
self.c_layout,
|
| 849 |
+
elem_ty_d=self.c_dtype,
|
| 850 |
+
elem_ty_acc=self.acc_dtype,
|
| 851 |
+
)
|
| 852 |
+
|
| 853 |
+
copy_atom_C = cute.make_copy_atom(
|
| 854 |
+
cute.nvgpu.warp.StMatrix8x8x16bOp(
|
| 855 |
+
self.c_layout.is_m_major_c(),
|
| 856 |
+
4,
|
| 857 |
+
),
|
| 858 |
+
self.c_dtype,
|
| 859 |
+
)
|
| 860 |
+
|
| 861 |
+
tiled_copy_C_Atom = cute.make_tiled_copy_C_atom(copy_atom_C, tiled_mma)
|
| 862 |
+
|
| 863 |
+
tiled_copy_r2s = cute.make_tiled_copy_S(
|
| 864 |
+
copy_atom_r2s,
|
| 865 |
+
tiled_copy_C_Atom,
|
| 866 |
+
)
|
| 867 |
+
|
| 868 |
+
# (R2S, R2S_M, R2S_N, PIPE_D)
|
| 869 |
+
thr_copy_r2s = tiled_copy_r2s.get_slice(tidx)
|
| 870 |
+
tRS_sD = thr_copy_r2s.partition_D(sC)
|
| 871 |
+
# (R2S, R2S_M, R2S_N)
|
| 872 |
+
tRS_rAcc = tiled_copy_r2s.retile(accumulators)
|
| 873 |
+
tRS_cD = thr_copy_r2s.partition_S(gC_coord)
|
| 874 |
+
|
| 875 |
+
# Allocate D registers.
|
| 876 |
+
rD_shape = cute.shape(thr_copy_r2s.partition_S(sC))
|
| 877 |
+
tRS_rD_layout = cute.make_layout(rD_shape[:3])
|
| 878 |
+
tRS_rD = cute.make_rmem_tensor_like(tRS_rD_layout, self.acc_dtype)
|
| 879 |
+
size_tRS_rD = cute.size(tRS_rD)
|
| 880 |
+
|
| 881 |
+
sepi_for_tma_partition = cute.group_modes(sC, 0, 2)
|
| 882 |
+
tCgC_for_tma_partition = cute.zipped_divide(gC_mnl, self.epi_tile)
|
| 883 |
+
|
| 884 |
+
bSG_sD, bSG_gD = cute.nvgpu.cpasync.tma_partition(
|
| 885 |
+
tma_atom_c,
|
| 886 |
+
0,
|
| 887 |
+
cute.make_layout(1),
|
| 888 |
+
sepi_for_tma_partition,
|
| 889 |
+
tCgC_for_tma_partition,
|
| 890 |
+
)
|
| 891 |
+
|
| 892 |
+
epi_tile_num = cute.size(tCgC_for_tma_partition, mode=[1])
|
| 893 |
+
epi_tile_shape = tCgC_for_tma_partition.shape[1]
|
| 894 |
+
epi_tile_layout = cute.make_layout(
|
| 895 |
+
epi_tile_shape, stride=(epi_tile_shape[1], 1)
|
| 896 |
+
)
|
| 897 |
+
|
| 898 |
+
# Initialize tma store c_pipeline
|
| 899 |
+
c_producer_group = pipeline.CooperativeGroup(
|
| 900 |
+
pipeline.Agent.Thread, self.threads_per_cta
|
| 901 |
+
)
|
| 902 |
+
c_pipeline = pipeline.PipelineTmaStore.create(
|
| 903 |
+
num_stages=self.epi_stage,
|
| 904 |
+
producer_group=c_producer_group,
|
| 905 |
+
)
|
| 906 |
+
|
| 907 |
+
for epi_idx in cutlass.range_constexpr(epi_tile_num):
|
| 908 |
+
# Copy from accumulators to D registers
|
| 909 |
+
for epi_v in cutlass.range_constexpr(size_tRS_rD):
|
| 910 |
+
tRS_rD[epi_v] = tRS_rAcc[epi_idx * size_tRS_rD + epi_v]
|
| 911 |
+
|
| 912 |
+
tRS_rD_out = cute.make_rmem_tensor_like(tRS_rD_layout, self.c_dtype)
|
| 913 |
+
for epi_v in cutlass.range_constexpr(size_tRS_rD):
|
| 914 |
+
register_idx = epi_idx * size_tRS_rD + epi_v
|
| 915 |
+
global_coord = tRS_cD[register_idx]
|
| 916 |
+
global_m = cute.crd2idx(global_coord[0], cute.make_layout(mC_mnl.shape[0]))
|
| 917 |
+
global_n = cute.crd2idx(global_coord[1], cute.make_layout(mC_mnl.shape[1]))
|
| 918 |
+
if global_m < cute.size(mScaleA) and global_n < cute.size(mScaleB):
|
| 919 |
+
value = tRS_rD[epi_v].to(cutlass.Float32)
|
| 920 |
+
value = mul_rn(mul_rn(value, mScaleA[global_m]), mScaleB[global_n])
|
| 921 |
+
value = value + mBias[global_n].to(cutlass.Float32)
|
| 922 |
+
tRS_rD_out[epi_v] = value.to(self.c_dtype)
|
| 923 |
+
else:
|
| 924 |
+
tRS_rD_out[epi_v] = self.c_dtype(0)
|
| 925 |
+
|
| 926 |
+
# Copy from D registers to shared memory
|
| 927 |
+
epi_buffer = epi_idx % cute.size(tRS_sD, mode=[3])
|
| 928 |
+
cute.copy(
|
| 929 |
+
tiled_copy_r2s, tRS_rD_out, tRS_sD[(None, None, None, epi_buffer)]
|
| 930 |
+
)
|
| 931 |
+
|
| 932 |
+
cute.arch.fence_proxy(
|
| 933 |
+
"async.shared",
|
| 934 |
+
space="cta",
|
| 935 |
+
)
|
| 936 |
+
# barrier for sync
|
| 937 |
+
pipeline.sync(barrier_id=1)
|
| 938 |
+
|
| 939 |
+
gmem_coord = epi_tile_layout.get_hier_coord(epi_idx)
|
| 940 |
+
# Copy from shared memory to global memory
|
| 941 |
+
if warp_idx == 0:
|
| 942 |
+
cute.copy(
|
| 943 |
+
tma_atom_c,
|
| 944 |
+
bSG_sD[(None, epi_buffer)],
|
| 945 |
+
bSG_gD[(None, gmem_coord)],
|
| 946 |
+
)
|
| 947 |
+
c_pipeline.producer_commit()
|
| 948 |
+
c_pipeline.producer_acquire()
|
| 949 |
+
|
| 950 |
+
pipeline.sync(barrier_id=1)
|
| 951 |
+
|
| 952 |
+
if warp_idx == 0:
|
| 953 |
+
c_pipeline.producer_tail()
|
| 954 |
+
|
| 955 |
+
return
|
| 956 |
+
|
| 957 |
+
@staticmethod
|
| 958 |
+
def _compute_stages(
|
| 959 |
+
tile_shape_mnk: tuple[int, int, int],
|
| 960 |
+
a_dtype: type[cutlass.Numeric],
|
| 961 |
+
b_dtype: type[cutlass.Numeric],
|
| 962 |
+
smem_capacity: int,
|
| 963 |
+
occupancy: int,
|
| 964 |
+
) -> tuple[int, int]:
|
| 965 |
+
"""Computes the number of stages for A/B/C operands based on heuristics.
|
| 966 |
+
|
| 967 |
+
:param tile_shape_mnk: The shape (M, N, K) of the CTA tile.
|
| 968 |
+
:type tile_shape_mnk: tuple[int, int, int]
|
| 969 |
+
:param a_dtype: Data type of operand A.
|
| 970 |
+
:type a_dtype: type[cutlass.Numeric]
|
| 971 |
+
:param b_dtype: Data type of operand B.
|
| 972 |
+
:type b_dtype: type[cutlass.Numeric]
|
| 973 |
+
:param smem_capacity: Total available shared memory capacity in bytes.
|
| 974 |
+
:type smem_capacity: int
|
| 975 |
+
:param occupancy: Target number of CTAs per SM (occupancy).
|
| 976 |
+
:type occupancy: int
|
| 977 |
+
|
| 978 |
+
:return: A tuple containing the computed number of stages for:
|
| 979 |
+
(A/B operand stages, epilogue stages)
|
| 980 |
+
:rtype: tuple[int, int]
|
| 981 |
+
"""
|
| 982 |
+
|
| 983 |
+
epi_stage = 4
|
| 984 |
+
# epi_smem will reuse smem ab.
|
| 985 |
+
epi_bytes = 0
|
| 986 |
+
|
| 987 |
+
a_shape = cute.slice_(tile_shape_mnk, (None, 0, None))
|
| 988 |
+
b_shape = cute.slice_(tile_shape_mnk, (0, None, None))
|
| 989 |
+
ab_bytes_per_stage = (
|
| 990 |
+
cute.size(a_shape) * a_dtype.width // 8
|
| 991 |
+
+ cute.size(b_shape) * b_dtype.width // 8
|
| 992 |
+
)
|
| 993 |
+
mbar_helpers_bytes = 1024
|
| 994 |
+
|
| 995 |
+
ab_stage = (
|
| 996 |
+
smem_capacity // occupancy - mbar_helpers_bytes - epi_bytes
|
| 997 |
+
) // ab_bytes_per_stage
|
| 998 |
+
return ab_stage, epi_stage
|
| 999 |
+
|
| 1000 |
+
@staticmethod
|
| 1001 |
+
def _make_smem_layouts(
|
| 1002 |
+
tile_shape_mnk: tuple[int, int, int],
|
| 1003 |
+
epi_tile: tuple[int, int],
|
| 1004 |
+
a_dtype: type[cutlass.Numeric],
|
| 1005 |
+
a_layout: utils.LayoutEnum,
|
| 1006 |
+
b_dtype: type[cutlass.Numeric],
|
| 1007 |
+
b_layout: utils.LayoutEnum,
|
| 1008 |
+
ab_stage: int,
|
| 1009 |
+
c_dtype: type[cutlass.Numeric],
|
| 1010 |
+
c_layout: utils.LayoutEnum,
|
| 1011 |
+
epi_stage: int,
|
| 1012 |
+
) -> tuple[cute.ComposedLayout, cute.ComposedLayout, cute.ComposedLayout]:
|
| 1013 |
+
"""Create shared memory layouts for A, B, and C tensors.
|
| 1014 |
+
|
| 1015 |
+
:param tile_shape_mnk: CTA tile shape (M,N,K)
|
| 1016 |
+
:type tile_shape_mnk: Tuple[int, int, int]
|
| 1017 |
+
:param epi_tile: Epilogue tile shape
|
| 1018 |
+
:type epi_tile: Tuple[int, int]
|
| 1019 |
+
:param a_dtype: Data type for matrix A
|
| 1020 |
+
:type a_dtype: type[cutlass.Numeric]
|
| 1021 |
+
:param a_layout: Layout enum for matrix A
|
| 1022 |
+
:type a_layout: utils.LayoutEnum
|
| 1023 |
+
:param b_dtype: Data type for matrix B
|
| 1024 |
+
:type b_dtype: type[cutlass.Numeric]
|
| 1025 |
+
:param b_layout: Layout enum for matrix B
|
| 1026 |
+
:type b_layout: utils.LayoutEnum
|
| 1027 |
+
:param ab_stage: Number of stages for A/B tensors
|
| 1028 |
+
:type ab_stage: int
|
| 1029 |
+
:param c_dtype: Data type for output matrix C
|
| 1030 |
+
:type c_dtype: type[cutlass.Numeric]
|
| 1031 |
+
:param c_layout: Layout enum for the output matrix C
|
| 1032 |
+
:type c_layout: utils.LayoutEnum
|
| 1033 |
+
:param epi_stage: Number of epilogue stages
|
| 1034 |
+
:type epi_stage: int
|
| 1035 |
+
|
| 1036 |
+
:return: Tuple of shared memory layouts for A, B, and C
|
| 1037 |
+
:rtype: Tuple[cute.ComposedLayout, cute.ComposedLayout, cute.ComposedLayout]
|
| 1038 |
+
"""
|
| 1039 |
+
a_smem_layout_staged = sm90_utils.make_smem_layout_a(
|
| 1040 |
+
a_layout,
|
| 1041 |
+
tile_shape_mnk,
|
| 1042 |
+
a_dtype,
|
| 1043 |
+
ab_stage,
|
| 1044 |
+
)
|
| 1045 |
+
|
| 1046 |
+
b_smem_layout_staged = sm90_utils.make_smem_layout_b(
|
| 1047 |
+
b_layout,
|
| 1048 |
+
tile_shape_mnk,
|
| 1049 |
+
b_dtype,
|
| 1050 |
+
ab_stage,
|
| 1051 |
+
)
|
| 1052 |
+
|
| 1053 |
+
epi_smem_layout_staged = sm90_utils.make_smem_layout_epi(
|
| 1054 |
+
c_dtype,
|
| 1055 |
+
c_layout,
|
| 1056 |
+
epi_tile,
|
| 1057 |
+
epi_stage,
|
| 1058 |
+
)
|
| 1059 |
+
|
| 1060 |
+
return a_smem_layout_staged, b_smem_layout_staged, epi_smem_layout_staged
|
| 1061 |
+
|
| 1062 |
+
@staticmethod
|
| 1063 |
+
def _compute_grid(
|
| 1064 |
+
c: cute.Tensor,
|
| 1065 |
+
tile_shape_mnk: tuple[int, int, int],
|
| 1066 |
+
cluster_shape_mn: tuple[int, int],
|
| 1067 |
+
) -> tuple[int, int, int]:
|
| 1068 |
+
"""Compute grid shape for the output tensor C.
|
| 1069 |
+
|
| 1070 |
+
:param c: The output tensor C
|
| 1071 |
+
:type c: cute.Tensor
|
| 1072 |
+
:param tile_shape_mnk: The shape (M, N, K) of the CTA tile.
|
| 1073 |
+
:type tile_shape_mnk: tuple[int, int, int]
|
| 1074 |
+
:param cluster_shape_mn: Shape of each cluster in M, N dimensions.
|
| 1075 |
+
:type cluster_shape_mn: tuple[int, int]
|
| 1076 |
+
|
| 1077 |
+
:return: Grid shape for kernel launch.
|
| 1078 |
+
:rtype: tuple[int, int, int]
|
| 1079 |
+
"""
|
| 1080 |
+
|
| 1081 |
+
c_shape = (tile_shape_mnk[0], tile_shape_mnk[1])
|
| 1082 |
+
gc = cute.zipped_divide(c, tiler=c_shape)
|
| 1083 |
+
cluster_shape_mnl = (*cluster_shape_mn, 1)
|
| 1084 |
+
clusters = cute.ceil_div(cute.get(gc.layout, mode=[1]).shape, cluster_shape_mnl)
|
| 1085 |
+
grid = tuple(x * y for x, y in zip(clusters, cluster_shape_mnl))
|
| 1086 |
+
return grid
|
| 1087 |
+
|
| 1088 |
+
@staticmethod
|
| 1089 |
+
def _make_tma_store_atoms_and_tensors(
|
| 1090 |
+
tensor_c: cute.Tensor,
|
| 1091 |
+
epi_smem_layout_staged: cute.ComposedLayout,
|
| 1092 |
+
epi_tile: tuple[int, int],
|
| 1093 |
+
) -> tuple[cute.CopyAtom, cute.Tensor]:
|
| 1094 |
+
"""Create TMA atoms and tensors for C tensor storage.
|
| 1095 |
+
|
| 1096 |
+
:param tensor_c: Output tensor C
|
| 1097 |
+
:type tensor_c: cute.Tensor
|
| 1098 |
+
:param epi_smem_layout_staged: Shared memory layout for epilogue
|
| 1099 |
+
:type epi_smem_layout_staged: cute.ComposedLayout
|
| 1100 |
+
:param epi_tile: Epilogue tile shape
|
| 1101 |
+
:type epi_tile: Tuple[int, int]
|
| 1102 |
+
|
| 1103 |
+
:return: TMA atom and tensor for C
|
| 1104 |
+
:rtype: Tuple[cute.CopyAtom, cute.Tensor]
|
| 1105 |
+
"""
|
| 1106 |
+
epi_smem_layout = cute.slice_(epi_smem_layout_staged, (None, None, 0))
|
| 1107 |
+
tma_atom_c, tma_tensor_c = cute.nvgpu.cpasync.make_tiled_tma_atom(
|
| 1108 |
+
cute.nvgpu.cpasync.CopyBulkTensorTileS2GOp(),
|
| 1109 |
+
tensor_c,
|
| 1110 |
+
epi_smem_layout,
|
| 1111 |
+
epi_tile,
|
| 1112 |
+
)
|
| 1113 |
+
|
| 1114 |
+
return tma_atom_c, tma_tensor_c
|
| 1115 |
+
|
| 1116 |
+
@staticmethod
|
| 1117 |
+
def _make_tma_atoms_and_tensors(
|
| 1118 |
+
tensor: cute.Tensor,
|
| 1119 |
+
smem_layout_staged: cute.ComposedLayout,
|
| 1120 |
+
smem_tile: tuple[int, int],
|
| 1121 |
+
mcast_dim: int,
|
| 1122 |
+
) -> tuple[cute.CopyAtom, cute.Tensor]:
|
| 1123 |
+
"""Create TMA atoms and tensors for input tensors.
|
| 1124 |
+
|
| 1125 |
+
:param tensor: Input tensor (A or B)
|
| 1126 |
+
:type tensor: cute.Tensor
|
| 1127 |
+
:param smem_layout_staged: Shared memory layout for the tensor
|
| 1128 |
+
:type smem_layout_staged: cute.ComposedLayout
|
| 1129 |
+
:param smem_tile: Shared memory tile shape
|
| 1130 |
+
:type smem_tile: Tuple[int, int]
|
| 1131 |
+
:param mcast_dim: Multicast dimension
|
| 1132 |
+
:type mcast_dim: int
|
| 1133 |
+
|
| 1134 |
+
:return: TMA atom and tensor
|
| 1135 |
+
:rtype: Tuple[cute.CopyAtom, cute.Tensor]
|
| 1136 |
+
"""
|
| 1137 |
+
op = (
|
| 1138 |
+
cute.nvgpu.cpasync.CopyBulkTensorTileG2SOp()
|
| 1139 |
+
if mcast_dim == 1
|
| 1140 |
+
else cute.nvgpu.cpasync.CopyBulkTensorTileG2SMulticastOp()
|
| 1141 |
+
)
|
| 1142 |
+
|
| 1143 |
+
smem_layout = cute.slice_(smem_layout_staged, (None, None, 0))
|
| 1144 |
+
tma_atom, tma_tensor = cute.nvgpu.cpasync.make_tiled_tma_atom(
|
| 1145 |
+
op,
|
| 1146 |
+
tensor,
|
| 1147 |
+
smem_layout,
|
| 1148 |
+
smem_tile,
|
| 1149 |
+
num_multicast=mcast_dim,
|
| 1150 |
+
)
|
| 1151 |
+
return tma_atom, tma_tensor
|
| 1152 |
+
|
| 1153 |
+
@staticmethod
|
| 1154 |
+
def is_valid_dtypes(
|
| 1155 |
+
a_dtype: Type[cutlass.Numeric],
|
| 1156 |
+
b_dtype: Type[cutlass.Numeric],
|
| 1157 |
+
acc_dtype: Type[cutlass.Numeric],
|
| 1158 |
+
c_dtype: Type[cutlass.Numeric],
|
| 1159 |
+
a_major: str,
|
| 1160 |
+
b_major: str,
|
| 1161 |
+
) -> bool:
|
| 1162 |
+
"""
|
| 1163 |
+
Check if the dtypes are valid
|
| 1164 |
+
|
| 1165 |
+
:param a_dtype: The data type of tensor A
|
| 1166 |
+
:type a_dtype: Type[cutlass.Numeric]
|
| 1167 |
+
:param b_dtype: The data type of tensor B
|
| 1168 |
+
:type b_dtype: Type[cutlass.Numeric]
|
| 1169 |
+
:param acc_dtype: The data type of the accumulator
|
| 1170 |
+
:type acc_dtype: Type[cutlass.Numeric]
|
| 1171 |
+
:param c_dtype: The data type of the output tensor
|
| 1172 |
+
:type c_dtype: Type[cutlass.Numeric]
|
| 1173 |
+
:param a_major: major mode of tensor A
|
| 1174 |
+
:type a_major: str
|
| 1175 |
+
:param b_major: major mode of tensor B
|
| 1176 |
+
:type b_major: str
|
| 1177 |
+
|
| 1178 |
+
:return: True if the dtypes are valid, False otherwise
|
| 1179 |
+
:rtype: bool
|
| 1180 |
+
"""
|
| 1181 |
+
is_valid = True
|
| 1182 |
+
|
| 1183 |
+
valid_ab_dtypes = {
|
| 1184 |
+
cutlass.Float16,
|
| 1185 |
+
cutlass.Float8E4M3FN,
|
| 1186 |
+
cutlass.Float8E5M2,
|
| 1187 |
+
cutlass.Uint8,
|
| 1188 |
+
cutlass.Int8,
|
| 1189 |
+
}
|
| 1190 |
+
if a_dtype not in valid_ab_dtypes:
|
| 1191 |
+
is_valid = False
|
| 1192 |
+
if b_dtype not in valid_ab_dtypes:
|
| 1193 |
+
is_valid = False
|
| 1194 |
+
|
| 1195 |
+
# make sure a_dtype == b_dtype for Float16
|
| 1196 |
+
if a_dtype.width == 16 and a_dtype != b_dtype:
|
| 1197 |
+
is_valid = False
|
| 1198 |
+
if a_dtype.width != b_dtype.width:
|
| 1199 |
+
is_valid = False
|
| 1200 |
+
if not a_dtype.is_same_kind(b_dtype):
|
| 1201 |
+
is_valid = False
|
| 1202 |
+
|
| 1203 |
+
# for 8-bit types, this implementation only supports k-major layout
|
| 1204 |
+
if (a_dtype.width == 8 and a_major != "k") or (
|
| 1205 |
+
b_dtype.width == 8 and b_major != "k"
|
| 1206 |
+
):
|
| 1207 |
+
is_valid = False
|
| 1208 |
+
|
| 1209 |
+
# Define compatibility mapping between accumulator type and AB type
|
| 1210 |
+
acc_ab_compatibility = {
|
| 1211 |
+
cutlass.Float32: {
|
| 1212 |
+
cutlass.Float16,
|
| 1213 |
+
cutlass.Float8E4M3FN,
|
| 1214 |
+
cutlass.Float8E5M2,
|
| 1215 |
+
},
|
| 1216 |
+
cutlass.Float16: {
|
| 1217 |
+
cutlass.Float16,
|
| 1218 |
+
cutlass.Float8E4M3FN,
|
| 1219 |
+
cutlass.Float8E5M2,
|
| 1220 |
+
},
|
| 1221 |
+
cutlass.Int32: {cutlass.Uint8, cutlass.Int8},
|
| 1222 |
+
}
|
| 1223 |
+
# Check compatibility between accumulator type and A type
|
| 1224 |
+
if a_dtype not in acc_ab_compatibility[acc_dtype]:
|
| 1225 |
+
is_valid = False
|
| 1226 |
+
|
| 1227 |
+
# Define compatibility mapping between accumulator type and C type
|
| 1228 |
+
acc_c_compatibility = {
|
| 1229 |
+
cutlass.Float32: {
|
| 1230 |
+
cutlass.Float32,
|
| 1231 |
+
cutlass.Float16,
|
| 1232 |
+
cutlass.Float8E4M3FN,
|
| 1233 |
+
cutlass.Float8E5M2,
|
| 1234 |
+
},
|
| 1235 |
+
cutlass.Float16: {
|
| 1236 |
+
cutlass.Float32,
|
| 1237 |
+
cutlass.Float16,
|
| 1238 |
+
cutlass.Float8E4M3FN,
|
| 1239 |
+
cutlass.Float8E5M2,
|
| 1240 |
+
},
|
| 1241 |
+
cutlass.Int32: {
|
| 1242 |
+
cutlass.Float32,
|
| 1243 |
+
cutlass.Float16,
|
| 1244 |
+
cutlass.BFloat16,
|
| 1245 |
+
cutlass.Int32,
|
| 1246 |
+
cutlass.Int8,
|
| 1247 |
+
cutlass.Uint8,
|
| 1248 |
+
},
|
| 1249 |
+
}
|
| 1250 |
+
# Check compatibility between accumulator type and C type
|
| 1251 |
+
if c_dtype not in acc_c_compatibility[acc_dtype]:
|
| 1252 |
+
is_valid = False
|
| 1253 |
+
|
| 1254 |
+
return is_valid
|
| 1255 |
+
|
| 1256 |
+
@staticmethod
|
| 1257 |
+
def is_valid_tensor_alignment(
|
| 1258 |
+
m: int,
|
| 1259 |
+
n: int,
|
| 1260 |
+
k: int,
|
| 1261 |
+
l: int, # noqa: E741 - CUTLASS uses L for the batch dimension.
|
| 1262 |
+
ab_dtype: Type[cutlass.Numeric],
|
| 1263 |
+
c_dtype: Type[cutlass.Numeric],
|
| 1264 |
+
a_major: str,
|
| 1265 |
+
b_major: str,
|
| 1266 |
+
c_major: str,
|
| 1267 |
+
) -> bool:
|
| 1268 |
+
"""
|
| 1269 |
+
Check if the tensor alignment is valid
|
| 1270 |
+
|
| 1271 |
+
:param m: The number of rows in the A tensor
|
| 1272 |
+
:type m: int
|
| 1273 |
+
:param n: The number of columns in the B tensor
|
| 1274 |
+
:type n: int
|
| 1275 |
+
:param k: The number of columns in the A tensor
|
| 1276 |
+
:type k: int
|
| 1277 |
+
:param l: The number of columns in the C tensor
|
| 1278 |
+
:type l: int
|
| 1279 |
+
:param ab_dtype: The data type of the A and B operands
|
| 1280 |
+
:type ab_dtype: Type[cutlass.Numeric]
|
| 1281 |
+
:param c_dtype: The data type of the output tensor
|
| 1282 |
+
:type c_dtype: Type[cutlass.Numeric]
|
| 1283 |
+
:param a_major: The major axis of the A tensor
|
| 1284 |
+
:type a_major: str
|
| 1285 |
+
:param b_major: The major axis of the B tensor
|
| 1286 |
+
:type b_major: str
|
| 1287 |
+
:param c_major: The major axis of the C tensor
|
| 1288 |
+
:type c_major: str
|
| 1289 |
+
|
| 1290 |
+
:return: True if the problem shape is valid, False otherwise
|
| 1291 |
+
:rtype: bool
|
| 1292 |
+
"""
|
| 1293 |
+
is_valid = True
|
| 1294 |
+
|
| 1295 |
+
def check_contigous_16B_alignment(dtype, is_mode0_major, tensor_shape):
|
| 1296 |
+
major_mode_idx = 0 if is_mode0_major else 1
|
| 1297 |
+
num_major_elements = tensor_shape[major_mode_idx]
|
| 1298 |
+
num_contiguous_elements = 16 * 8 // dtype.width
|
| 1299 |
+
return num_major_elements % num_contiguous_elements == 0
|
| 1300 |
+
|
| 1301 |
+
if (
|
| 1302 |
+
not check_contigous_16B_alignment(ab_dtype, a_major == "m", (m, k, l))
|
| 1303 |
+
or not check_contigous_16B_alignment(ab_dtype, b_major == "n", (n, k, l))
|
| 1304 |
+
or not check_contigous_16B_alignment(c_dtype, c_major == "m", (m, n, l))
|
| 1305 |
+
):
|
| 1306 |
+
is_valid = False
|
| 1307 |
+
return is_valid
|
runtime/clef_text_cute.py
ADDED
|
@@ -0,0 +1,341 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Inference-only CuTe pointwise fusions for CLEF's dense text backbone."""
|
| 2 |
+
|
| 3 |
+
import cuda.bindings.driver as cuda
|
| 4 |
+
import cutlass
|
| 5 |
+
import cutlass.cute as cute
|
| 6 |
+
from cutlass.cute.runtime import make_fake_stream, make_fake_tensor
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from clef_vision_norm import mul_rn
|
| 10 |
+
|
| 11 |
+
_CACHE = {}
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class TextRMSNorm:
|
| 15 |
+
"""Match ATen's FP32 mean reduction: four accumulators, shuffle 1..16."""
|
| 16 |
+
|
| 17 |
+
def __init__(self, width, eps, residual):
|
| 18 |
+
self.width, self.eps, self.residual = width, eps, residual
|
| 19 |
+
|
| 20 |
+
@cute.jit
|
| 21 |
+
def __call__(self, x: cute.Tensor, skip: cute.Tensor, weight: cute.Tensor,
|
| 22 |
+
summed: cute.Tensor, output: cute.Tensor, stream: cuda.CUstream):
|
| 23 |
+
self.kernel(x, skip, weight, summed, output).launch(
|
| 24 |
+
grid=(cute.ceil_div(x.shape[0], 4), 1, 1), block=(128, 1, 1), stream=stream)
|
| 25 |
+
|
| 26 |
+
@cute.kernel
|
| 27 |
+
def kernel(self, x: cute.Tensor, skip: cute.Tensor, weight: cute.Tensor,
|
| 28 |
+
summed: cute.Tensor, output: cute.Tensor):
|
| 29 |
+
tid, _, _ = cute.arch.thread_idx()
|
| 30 |
+
block, _, _ = cute.arch.block_idx()
|
| 31 |
+
lane, row = tid % 32, block * 4 + tid // 32
|
| 32 |
+
if row < x.shape[0]:
|
| 33 |
+
copy = cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), x.element_type, num_bits_per_copy=64)
|
| 34 |
+
accum = cute.make_rmem_tensor((4,), cutlass.Float32)
|
| 35 |
+
for i in cutlass.range_constexpr(4):
|
| 36 |
+
accum[i] = 0.0
|
| 37 |
+
for step in cutlass.range_constexpr(self.width // 128):
|
| 38 |
+
col = cute.assume(lane * 4 + step * 128, divby=4)
|
| 39 |
+
offset = cute.assume(row * self.width + col, divby=4)
|
| 40 |
+
layout = cute.make_layout((4,))
|
| 41 |
+
xr = cute.make_rmem_tensor((4,), x.element_type)
|
| 42 |
+
cute.copy(copy, cute.make_tensor(x.iterator + offset, layout), xr)
|
| 43 |
+
if cutlass.const_expr(self.residual):
|
| 44 |
+
sr = cute.make_rmem_tensor((4,), x.element_type)
|
| 45 |
+
cute.copy(copy, cute.make_tensor(skip.iterator + offset, layout), sr)
|
| 46 |
+
for i in cutlass.range_constexpr(4):
|
| 47 |
+
xr[i] = (xr[i].to(cutlass.Float32) + sr[i].to(cutlass.Float32)).to(x.element_type)
|
| 48 |
+
for i in cutlass.range_constexpr(4):
|
| 49 |
+
value = xr[i].to(cutlass.Float32)
|
| 50 |
+
accum[i] = accum[i] + mul_rn(value, value)
|
| 51 |
+
total = ((accum[0] + accum[1]) + accum[2]) + accum[3]
|
| 52 |
+
for step in cutlass.range_constexpr(5):
|
| 53 |
+
total = total + cute.arch.shuffle_sync_down(total, 1 << step)
|
| 54 |
+
total = cute.arch.shuffle_sync(total, 0)
|
| 55 |
+
mean = mul_rn(total, cutlass.Float32(1.0 / self.width))
|
| 56 |
+
inv = cute.math.rsqrt(mean + self.eps, fastmath=True)
|
| 57 |
+
for step in cutlass.range_constexpr(self.width // 128):
|
| 58 |
+
col = cute.assume(lane * 4 + step * 128, divby=4)
|
| 59 |
+
offset = cute.assume(row * self.width + col, divby=4)
|
| 60 |
+
layout = cute.make_layout((4,))
|
| 61 |
+
xr = cute.make_rmem_tensor((4,), x.element_type)
|
| 62 |
+
wr = cute.make_rmem_tensor((4,), x.element_type)
|
| 63 |
+
cute.copy(copy, cute.make_tensor(x.iterator + offset, layout), xr)
|
| 64 |
+
cute.copy(copy, cute.make_tensor(weight.iterator + col, layout), wr)
|
| 65 |
+
if cutlass.const_expr(self.residual):
|
| 66 |
+
sr = cute.make_rmem_tensor((4,), x.element_type)
|
| 67 |
+
cute.copy(copy, cute.make_tensor(skip.iterator + offset, layout), sr)
|
| 68 |
+
for i in cutlass.range_constexpr(4):
|
| 69 |
+
xr[i] = (xr[i].to(cutlass.Float32) + sr[i].to(cutlass.Float32)).to(x.element_type)
|
| 70 |
+
cute.copy(copy, xr, cute.make_tensor(summed.iterator + offset, layout))
|
| 71 |
+
result = cute.make_rmem_tensor((4,), x.element_type)
|
| 72 |
+
for i in cutlass.range_constexpr(4):
|
| 73 |
+
value = mul_rn(xr[i].to(cutlass.Float32), inv)
|
| 74 |
+
result[i] = mul_rn(value, 1.0 + wr[i].to(cutlass.Float32)).to(x.element_type)
|
| 75 |
+
cute.copy(copy, result, cute.make_tensor(output.iterator + offset, layout))
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
class TextSwiGLU:
|
| 79 |
+
@cute.jit
|
| 80 |
+
def __call__(self, gate: cute.Tensor, up: cute.Tensor, output: cute.Tensor, stream: cuda.CUstream):
|
| 81 |
+
self.kernel(gate, up, output).launch(
|
| 82 |
+
grid=(cute.ceil_div(cute.size(gate), 2048), 1, 1), block=(256, 1, 1), stream=stream)
|
| 83 |
+
|
| 84 |
+
@cute.kernel
|
| 85 |
+
def kernel(self, gate: cute.Tensor, up: cute.Tensor, output: cute.Tensor):
|
| 86 |
+
tid, _, _ = cute.arch.thread_idx()
|
| 87 |
+
block, _, _ = cute.arch.block_idx()
|
| 88 |
+
offset = cute.assume((block * 256 + tid) * 8, divby=8)
|
| 89 |
+
if offset < cute.size(gate):
|
| 90 |
+
layout = cute.make_layout((8,))
|
| 91 |
+
copy = cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), gate.element_type, num_bits_per_copy=128)
|
| 92 |
+
gr = cute.make_rmem_tensor((8,), gate.element_type)
|
| 93 |
+
ur = cute.make_rmem_tensor((8,), gate.element_type)
|
| 94 |
+
result = cute.make_rmem_tensor((8,), gate.element_type)
|
| 95 |
+
cute.copy(copy, cute.make_tensor(gate.iterator + offset, layout), gr)
|
| 96 |
+
cute.copy(copy, cute.make_tensor(up.iterator + offset, layout), ur)
|
| 97 |
+
for i in cutlass.range_constexpr(8):
|
| 98 |
+
value = gr[i].to(cutlass.Float32)
|
| 99 |
+
activated = value / (1.0 + cute.math.exp(-value))
|
| 100 |
+
activated = activated.to(gate.element_type).to(cutlass.Float32)
|
| 101 |
+
result[i] = mul_rn(activated, ur[i].to(cutlass.Float32)).to(gate.element_type)
|
| 102 |
+
cute.copy(copy, result, cute.make_tensor(output.iterator + offset, layout))
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
class DeltaGates:
|
| 106 |
+
def __init__(self, heads):
|
| 107 |
+
self.heads = heads
|
| 108 |
+
|
| 109 |
+
@cute.jit
|
| 110 |
+
def __call__(self, a: cute.Tensor, b: cute.Tensor, bias: cute.Tensor, coefficient: cute.Tensor,
|
| 111 |
+
beta: cute.Tensor, g: cute.Tensor, stream: cuda.CUstream):
|
| 112 |
+
self.kernel(a, b, bias, coefficient, beta, g).launch(
|
| 113 |
+
grid=(cute.ceil_div(cute.size(a), 1024), 1, 1), block=(256, 1, 1), stream=stream)
|
| 114 |
+
|
| 115 |
+
@cute.kernel
|
| 116 |
+
def kernel(self, a: cute.Tensor, b: cute.Tensor, bias: cute.Tensor, coefficient: cute.Tensor,
|
| 117 |
+
beta: cute.Tensor, g: cute.Tensor):
|
| 118 |
+
tid, _, _ = cute.arch.thread_idx()
|
| 119 |
+
block, _, _ = cute.arch.block_idx()
|
| 120 |
+
offset = cute.assume((block * 256 + tid) * 4, divby=4)
|
| 121 |
+
head = cute.assume(offset % self.heads, divby=4)
|
| 122 |
+
if offset < cute.size(a):
|
| 123 |
+
layout = cute.make_layout((4,))
|
| 124 |
+
bf16_copy = cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), a.element_type, num_bits_per_copy=64)
|
| 125 |
+
fp32_copy = cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), coefficient.element_type, num_bits_per_copy=128)
|
| 126 |
+
ar, br, dr, result_beta = [cute.make_rmem_tensor((4,), a.element_type) for _ in range(4)]
|
| 127 |
+
cr, result_g = [cute.make_rmem_tensor((4,), coefficient.element_type) for _ in range(2)]
|
| 128 |
+
cute.copy(bf16_copy, cute.make_tensor(a.iterator + offset, layout), ar)
|
| 129 |
+
cute.copy(bf16_copy, cute.make_tensor(b.iterator + offset, layout), br)
|
| 130 |
+
cute.copy(bf16_copy, cute.make_tensor(bias.iterator + head, layout), dr)
|
| 131 |
+
cute.copy(fp32_copy, cute.make_tensor(coefficient.iterator + head, layout), cr)
|
| 132 |
+
for i in cutlass.range_constexpr(4):
|
| 133 |
+
value = ar[i].to(cutlass.Float32) + dr[i].to(cutlass.Float32)
|
| 134 |
+
softplus = value
|
| 135 |
+
if value <= 20.0:
|
| 136 |
+
softplus = cute.math.log1p(cute.math.exp(value))
|
| 137 |
+
result_g[i] = mul_rn(cr[i], softplus)
|
| 138 |
+
result_beta[i] = (1.0 / (1.0 + cute.math.exp(-br[i].to(cutlass.Float32)))).to(a.element_type)
|
| 139 |
+
cute.copy(bf16_copy, result_beta, cute.make_tensor(beta.iterator + offset, layout))
|
| 140 |
+
cute.copy(fp32_copy, result_g, cute.make_tensor(g.iterator + offset, layout))
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def _validate(*tensors):
|
| 144 |
+
device = tensors[0].device
|
| 145 |
+
if torch.is_grad_enabled():
|
| 146 |
+
raise RuntimeError("CLEF text CuTe kernels are inference-only")
|
| 147 |
+
if any(not t.is_cuda or t.device != device or t.dtype != torch.bfloat16
|
| 148 |
+
or not t.is_contiguous() or t.data_ptr() % 16 for t in tensors):
|
| 149 |
+
raise ValueError("Expected aligned, colocated, contiguous CUDA BF16 tensors")
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def rms_norm(x, norm, skip=None):
|
| 153 |
+
_validate(x, norm.weight, *(() if skip is None else (skip,)))
|
| 154 |
+
width = x.shape[-1]
|
| 155 |
+
rows = x.numel() // width
|
| 156 |
+
if width not in {256, 5120} or rows < 16 or norm.weight.shape != (width,):
|
| 157 |
+
raise ValueError("The exact CLEF RMSNorm path requires width 256/5120 and at least 16 rows")
|
| 158 |
+
if skip is not None and skip.shape != x.shape:
|
| 159 |
+
raise ValueError("Residual shape must match the input")
|
| 160 |
+
key = ("rms", x.device.index, rows, width, norm.eps, skip is not None)
|
| 161 |
+
fn = _CACHE.get(key)
|
| 162 |
+
if fn is None:
|
| 163 |
+
kwargs = {"memspace": cute.AddressSpace.gmem, "assumed_align": 16}
|
| 164 |
+
matrix = make_fake_tensor(cutlass.BFloat16, (rows, width), stride=(width, 1), **kwargs)
|
| 165 |
+
weight = make_fake_tensor(cutlass.BFloat16, (width,), stride=(1,), **kwargs)
|
| 166 |
+
fn = cute.compile(TextRMSNorm(width, norm.eps, skip is not None), matrix, matrix,
|
| 167 |
+
weight, matrix, matrix, make_fake_stream(use_tvm_ffi_env_stream=True),
|
| 168 |
+
options="--enable-tvm-ffi")
|
| 169 |
+
_CACHE[key] = fn
|
| 170 |
+
output = torch.empty_like(x)
|
| 171 |
+
summed = torch.empty_like(x) if skip is not None else x
|
| 172 |
+
fn(x.view(rows, width), (skip if skip is not None else x).view(rows, width), norm.weight,
|
| 173 |
+
summed.view(rows, width), output.view(rows, width))
|
| 174 |
+
return (summed, output) if skip is not None else output
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def swiglu(gate, up):
|
| 178 |
+
_validate(gate, up)
|
| 179 |
+
if gate.shape != up.shape or gate.numel() == 0 or gate.numel() % 8:
|
| 180 |
+
raise ValueError("Expected equal nonempty SwiGLU inputs with a multiple of eight elements")
|
| 181 |
+
key = ("swiglu", gate.device.index, gate.numel())
|
| 182 |
+
fn = _CACHE.get(key)
|
| 183 |
+
if fn is None:
|
| 184 |
+
tensor = make_fake_tensor(cutlass.BFloat16, (gate.numel(),), stride=(1,),
|
| 185 |
+
memspace=cute.AddressSpace.gmem, assumed_align=16)
|
| 186 |
+
fn = cute.compile(TextSwiGLU(), tensor, tensor, tensor,
|
| 187 |
+
make_fake_stream(use_tvm_ffi_env_stream=True), options="--enable-tvm-ffi")
|
| 188 |
+
_CACHE[key] = fn
|
| 189 |
+
output = torch.empty_like(gate)
|
| 190 |
+
fn(gate.flatten(), up.flatten(), output.flatten())
|
| 191 |
+
return output
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def delta_gates(a, b, bias, coefficient):
|
| 195 |
+
_validate(a, b, bias)
|
| 196 |
+
heads = bias.numel()
|
| 197 |
+
if (a.shape != b.shape or a.shape[-1] != heads or heads % 4
|
| 198 |
+
or coefficient.shape != bias.shape or coefficient.dtype != torch.float32
|
| 199 |
+
or coefficient.device != a.device or not coefficient.is_contiguous()):
|
| 200 |
+
raise ValueError("Expected matching delta gate inputs and FP32 decay coefficients")
|
| 201 |
+
key = ("gates", a.device.index, a.numel(), heads)
|
| 202 |
+
fn = _CACHE.get(key)
|
| 203 |
+
if fn is None:
|
| 204 |
+
kwargs = {"memspace": cute.AddressSpace.gmem, "assumed_align": 16}
|
| 205 |
+
inp = make_fake_tensor(cutlass.BFloat16, (a.numel(),), stride=(1,), **kwargs)
|
| 206 |
+
dt = make_fake_tensor(cutlass.BFloat16, (heads,), stride=(1,), **kwargs)
|
| 207 |
+
coef = make_fake_tensor(cutlass.Float32, (heads,), stride=(1,), **kwargs)
|
| 208 |
+
gout = make_fake_tensor(cutlass.Float32, (a.numel(),), stride=(1,), **kwargs)
|
| 209 |
+
fn = cute.compile(DeltaGates(heads), inp, inp, dt, coef, inp, gout,
|
| 210 |
+
make_fake_stream(use_tvm_ffi_env_stream=True), options="--enable-tvm-ffi")
|
| 211 |
+
_CACHE[key] = fn
|
| 212 |
+
beta, g = torch.empty_like(b), torch.empty_like(a, dtype=torch.float32)
|
| 213 |
+
fn(a.flatten(), b.flatten(), bias, coefficient, beta.flatten(), g.flatten())
|
| 214 |
+
return beta, g
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def patch_text(text, *, rms=True, activation=True, residual=True, gva=False, gates=False, heads=False,
|
| 218 |
+
share_mlp_quant=False, share_attn_quant=False):
|
| 219 |
+
"""Patch immutable evaluated BF16 text weights; return a restoration callable."""
|
| 220 |
+
if text.training or text.config.hidden_size != 5120:
|
| 221 |
+
raise ValueError("Expected CLEF's evaluated 5120-wide text backbone")
|
| 222 |
+
if text.config.hidden_act != "silu":
|
| 223 |
+
raise ValueError("CLEF text fusion requires SiLU activation")
|
| 224 |
+
if gva:
|
| 225 |
+
for layer in text.layers:
|
| 226 |
+
rule = getattr(getattr(layer, "linear_attn", None), "chunk_gated_delta_rule", None)
|
| 227 |
+
if getattr(rule, "__name__", "") == "torch_chunk_gated_delta_rule":
|
| 228 |
+
raise ValueError("Native grouped-value attention requires FLA")
|
| 229 |
+
originals = []
|
| 230 |
+
if share_mlp_quant or share_attn_quant:
|
| 231 |
+
from clef_text_int8 import shared_linear_pair
|
| 232 |
+
stats = {"rms_calls": 0, "swiglu_calls": 0, "residual_calls": 0, "delta_calls": 0,
|
| 233 |
+
"shared_quant_calls": 0, "shared_attn_quant_calls": 0, "fallbacks": 0}
|
| 234 |
+
|
| 235 |
+
def replace(module, forward):
|
| 236 |
+
originals.append((module, module.forward))
|
| 237 |
+
module.forward = forward
|
| 238 |
+
|
| 239 |
+
def supported(x):
|
| 240 |
+
return (not text.training and not torch.is_grad_enabled() and x.is_cuda
|
| 241 |
+
and x.dtype == torch.bfloat16 and x.numel() // x.shape[-1] >= 16)
|
| 242 |
+
|
| 243 |
+
if rms:
|
| 244 |
+
for norm in text.modules():
|
| 245 |
+
if norm.__class__.__name__ != "Qwen3_5RMSNorm" or norm.weight.numel() not in ({256, 5120} if heads else {5120}):
|
| 246 |
+
continue
|
| 247 |
+
original = norm.forward
|
| 248 |
+
def forward(x, norm=norm, original=original):
|
| 249 |
+
if supported(x):
|
| 250 |
+
stats["rms_calls"] += 1
|
| 251 |
+
return rms_norm(x.contiguous(), norm)
|
| 252 |
+
stats["fallbacks"] += 1
|
| 253 |
+
return original(x)
|
| 254 |
+
replace(norm, forward)
|
| 255 |
+
for layer in text.layers:
|
| 256 |
+
if (gva or gates or share_attn_quant) and layer.layer_type == "linear_attention":
|
| 257 |
+
attn = layer.linear_attn
|
| 258 |
+
original_attn = attn.forward
|
| 259 |
+
with torch.inference_mode():
|
| 260 |
+
coefficient = -attn.A_log.float().exp()
|
| 261 |
+
def forward(hidden_states, cache_params=None, attention_mask=None, attn=attn,
|
| 262 |
+
coefficient=coefficient, original=original_attn, **kwargs):
|
| 263 |
+
if cache_params is not None or attn.training or torch.is_grad_enabled() or hidden_states.dtype != torch.bfloat16:
|
| 264 |
+
stats["fallbacks"] += 1
|
| 265 |
+
return original(hidden_states, cache_params=cache_params, attention_mask=attention_mask, **kwargs)
|
| 266 |
+
from transformers.models.qwen3_5.modeling_qwen3_5 import apply_mask_to_padding_states
|
| 267 |
+
hidden_states = apply_mask_to_padding_states(hidden_states, attention_mask)
|
| 268 |
+
batch, seq, _ = hidden_states.shape
|
| 269 |
+
if share_attn_quant:
|
| 270 |
+
mixed, z, shared = shared_linear_pair(hidden_states, attn.in_proj_qkv, attn.in_proj_z)
|
| 271 |
+
stats["shared_attn_quant_calls"] += int(shared)
|
| 272 |
+
else:
|
| 273 |
+
mixed, z = attn.in_proj_qkv(hidden_states), attn.in_proj_z(hidden_states)
|
| 274 |
+
mixed = mixed.transpose(1, 2)
|
| 275 |
+
z = z.reshape(batch, seq, -1, attn.head_v_dim)
|
| 276 |
+
b, a = attn.in_proj_b(hidden_states), attn.in_proj_a(hidden_states)
|
| 277 |
+
if attn.causal_conv1d_fn is not None:
|
| 278 |
+
mixed = attn.causal_conv1d_fn(x=mixed, weight=attn.conv1d.weight.squeeze(1),
|
| 279 |
+
bias=attn.conv1d.bias, activation=attn.activation,
|
| 280 |
+
seq_idx=kwargs.get("seq_idx"))
|
| 281 |
+
else:
|
| 282 |
+
mixed = torch.nn.functional.silu(attn.conv1d(mixed)[:, :, :seq])
|
| 283 |
+
q, k, v = mixed.transpose(1, 2).split((attn.key_dim, attn.key_dim, attn.value_dim), dim=-1)
|
| 284 |
+
q, k = [t.reshape(batch, seq, -1, attn.head_k_dim) for t in (q, k)]
|
| 285 |
+
v = v.reshape(batch, seq, -1, attn.head_v_dim)
|
| 286 |
+
if gates:
|
| 287 |
+
beta, g = delta_gates(a, b, attn.dt_bias, coefficient)
|
| 288 |
+
else:
|
| 289 |
+
beta = b.sigmoid()
|
| 290 |
+
g = -attn.A_log.float().exp() * torch.nn.functional.softplus(a.float() + attn.dt_bias)
|
| 291 |
+
ratio = attn.num_v_heads // attn.num_k_heads
|
| 292 |
+
if not gva and ratio > 1:
|
| 293 |
+
q, k = [t.repeat_interleave(ratio, dim=2) for t in (q, k)]
|
| 294 |
+
core, _ = attn.chunk_gated_delta_rule(q, k, v, g=g, beta=beta, initial_state=None,
|
| 295 |
+
output_final_state=False, use_qk_l2norm_in_kernel=True,
|
| 296 |
+
cu_seqlens=kwargs.get("cu_seq_lens_q"))
|
| 297 |
+
core = attn.norm(core.reshape(-1, attn.head_v_dim), z.reshape(-1, attn.head_v_dim))
|
| 298 |
+
stats["delta_calls"] += 1
|
| 299 |
+
return attn.out_proj(core.reshape(batch, seq, -1))
|
| 300 |
+
replace(attn, forward)
|
| 301 |
+
if activation:
|
| 302 |
+
mlp = layer.mlp
|
| 303 |
+
original_mlp = mlp.forward
|
| 304 |
+
def forward(x, mlp=mlp, original=original_mlp):
|
| 305 |
+
if not supported(x) or mlp.training:
|
| 306 |
+
stats["fallbacks"] += 1
|
| 307 |
+
return original(x)
|
| 308 |
+
stats["swiglu_calls"] += 1
|
| 309 |
+
gate, up = mlp.gate_proj, mlp.up_proj
|
| 310 |
+
if share_mlp_quant:
|
| 311 |
+
gate_output, up_output, shared = shared_linear_pair(x, gate, up)
|
| 312 |
+
stats["shared_quant_calls"] += int(shared)
|
| 313 |
+
else:
|
| 314 |
+
gate_output, up_output = gate(x), up(x)
|
| 315 |
+
return mlp.down_proj(swiglu(gate_output, up_output))
|
| 316 |
+
replace(mlp, forward)
|
| 317 |
+
if rms and residual:
|
| 318 |
+
original = layer.forward
|
| 319 |
+
def forward(hidden_states, position_embeddings, attention_mask=None, position_ids=None,
|
| 320 |
+
past_key_values=None, layer=layer, original=original, **kwargs):
|
| 321 |
+
if not supported(hidden_states) or past_key_values is not None or kwargs.get("use_cache"):
|
| 322 |
+
stats["fallbacks"] += 1
|
| 323 |
+
return original(hidden_states, position_embeddings, attention_mask, position_ids, past_key_values, **kwargs)
|
| 324 |
+
normalized = layer.input_layernorm(hidden_states)
|
| 325 |
+
if layer.layer_type == "linear_attention":
|
| 326 |
+
branch = layer.linear_attn(hidden_states=normalized, cache_params=None,
|
| 327 |
+
attention_mask=attention_mask, **kwargs)
|
| 328 |
+
else:
|
| 329 |
+
branch, _ = layer.self_attn(hidden_states=normalized, position_embeddings=position_embeddings,
|
| 330 |
+
attention_mask=attention_mask, position_ids=position_ids,
|
| 331 |
+
past_key_values=None, **kwargs)
|
| 332 |
+
summed, normalized = rms_norm(hidden_states.contiguous(), layer.post_attention_layernorm, branch.contiguous())
|
| 333 |
+
stats["residual_calls"] += 1
|
| 334 |
+
return summed + layer.mlp(normalized)
|
| 335 |
+
replace(layer, forward)
|
| 336 |
+
|
| 337 |
+
def restore():
|
| 338 |
+
for module, original in reversed(originals):
|
| 339 |
+
module.forward = original
|
| 340 |
+
restore.statistics = stats
|
| 341 |
+
return restore
|
runtime/clef_text_int8.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Experimental CLEF W8A8 ConvRot with the H3 SM90 scaled-GEMM runtime."""
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
_CACHE = {}
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def scaled_int8_mm(x, weight, input_scale, weight_scale, bias, *, tile=(128, 256), cluster=(1, 1)):
|
| 9 |
+
import cutlass
|
| 10 |
+
import cutlass.cute as cute
|
| 11 |
+
from cutlass.cute.runtime import make_fake_stream, make_fake_tensor
|
| 12 |
+
from clef_int8_cutlass import HopperWgmmaGemmKernel
|
| 13 |
+
|
| 14 |
+
if torch.is_grad_enabled() or not x.is_cuda or torch.cuda.get_device_capability(x.device) != (9, 0):
|
| 15 |
+
raise ValueError("CLEF ConvRot GEMM requires inference mode and SM90")
|
| 16 |
+
if x.ndim != 2 or weight.ndim != 2:
|
| 17 |
+
raise ValueError("Expected two-dimensional INT8 matrices")
|
| 18 |
+
m, k = x.shape
|
| 19 |
+
n = weight.shape[0]
|
| 20 |
+
if (x.dtype != torch.int8 or weight.dtype != torch.int8 or weight.shape[1] != k
|
| 21 |
+
or not x.is_contiguous() or not weight.is_contiguous() or not m or not n or not k
|
| 22 |
+
or k % 16 or n % 8):
|
| 23 |
+
raise ValueError("Expected aligned INT8 row-major matrices")
|
| 24 |
+
tensors = (weight, input_scale, weight_scale, bias)
|
| 25 |
+
if any(t.device != x.device or not t.is_contiguous() or t.data_ptr() % 16 for t in tensors) or x.data_ptr() % 16:
|
| 26 |
+
raise ValueError("INT8 GEMM tensors must be colocated, contiguous, and 16-byte aligned")
|
| 27 |
+
if (input_scale.dtype != torch.float32 or input_scale.numel() != m
|
| 28 |
+
or weight_scale.dtype != torch.float32 or weight_scale.numel() != n
|
| 29 |
+
or bias.dtype != torch.bfloat16 or bias.shape != (n,)):
|
| 30 |
+
raise ValueError("Expected FP32 row/column scales and a BF16 output bias")
|
| 31 |
+
tile, cluster = tuple(tile), tuple(cluster)
|
| 32 |
+
if tile not in {(64, 128), (128, 128), (128, 256)} or cluster not in {(1, 1), (2, 1), (1, 2)}:
|
| 33 |
+
raise ValueError("Unsupported CLEF INT8 tile/cluster configuration")
|
| 34 |
+
key = (x.device.index, n, k, tile, cluster)
|
| 35 |
+
fn = _CACHE.get(key)
|
| 36 |
+
if fn is None:
|
| 37 |
+
dynamic_m = cute.sym_int()
|
| 38 |
+
kwargs = {"memspace": cute.AddressSpace.gmem, "assumed_align": 16}
|
| 39 |
+
a = make_fake_tensor(cutlass.Int8, (dynamic_m, k, 1), stride=(k, 1, 1), **kwargs)
|
| 40 |
+
b = make_fake_tensor(cutlass.Int8, (n, k, 1), stride=(k, 1, 1), **kwargs)
|
| 41 |
+
c = make_fake_tensor(cutlass.BFloat16, (dynamic_m, n, 1), stride=(n, 1, 1), **kwargs)
|
| 42 |
+
sa = make_fake_tensor(cutlass.Float32, (dynamic_m,), stride=(1,), **kwargs)
|
| 43 |
+
sb = make_fake_tensor(cutlass.Float32, (n,), stride=(1,), **kwargs)
|
| 44 |
+
bias_tensor = make_fake_tensor(cutlass.BFloat16, (n,), stride=(1,), **kwargs)
|
| 45 |
+
fn = cute.compile(HopperWgmmaGemmKernel(cutlass.Int32, tile, cluster),
|
| 46 |
+
a, b, c, sa, sb, bias_tensor, make_fake_stream(use_tvm_ffi_env_stream=True),
|
| 47 |
+
options="--enable-tvm-ffi")
|
| 48 |
+
_CACHE[key] = fn
|
| 49 |
+
output = torch.empty((m, n), device=x.device, dtype=torch.bfloat16)
|
| 50 |
+
fn(x.view(m, k, 1), weight.view(n, k, 1), output.view(m, n, 1),
|
| 51 |
+
input_scale.flatten(), weight_scale.flatten(), bias)
|
| 52 |
+
return output
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class ConvRotLinear(torch.nn.Module):
|
| 56 |
+
"""Immutable inference projection; rotation/quantization happens once per call."""
|
| 57 |
+
|
| 58 |
+
def __init__(self, linear, *, group_size=256, backend="cute", tile=(128, 256), cluster=(1, 1)):
|
| 59 |
+
super().__init__()
|
| 60 |
+
from comfy_kitchen import _cuda_backend
|
| 61 |
+
|
| 62 |
+
if (group_size != 256 or backend not in {"cute", "torch"} or linear.training
|
| 63 |
+
or torch.is_grad_enabled() or not linear.weight.is_cuda
|
| 64 |
+
or linear.weight.dtype != torch.bfloat16 or linear.in_features % group_size
|
| 65 |
+
or (linear.bias is not None and linear.bias.dtype != torch.bfloat16)):
|
| 66 |
+
raise ValueError("Expected BF16 weights divisible by the ConvRot group")
|
| 67 |
+
self.in_features, self.out_features = linear.in_features, linear.out_features
|
| 68 |
+
self.group_size, self.backend, self.tile, self.cluster = group_size, backend, tile, cluster
|
| 69 |
+
weight, scale = _cuda_backend.quantize_int8_rowwise_convrot64(linear.weight.detach().contiguous(), group_size)
|
| 70 |
+
self.register_buffer("weight", weight)
|
| 71 |
+
self.register_buffer("weight_scale", scale.flatten())
|
| 72 |
+
bias = linear.bias.detach().clone() if linear.bias is not None else torch.zeros(self.out_features, device=weight.device, dtype=torch.bfloat16)
|
| 73 |
+
self.register_buffer("bias", bias)
|
| 74 |
+
self.eval()
|
| 75 |
+
|
| 76 |
+
def quantize_input(self, x):
|
| 77 |
+
from comfy_kitchen import _cuda_backend
|
| 78 |
+
|
| 79 |
+
if (self.training or torch.is_grad_enabled() or x.dtype != torch.bfloat16
|
| 80 |
+
or x.device != self.weight.device or x.ndim < 1 or x.shape[-1] != self.in_features
|
| 81 |
+
or not x.numel()):
|
| 82 |
+
raise ValueError("CLEF ConvRot linears are BF16-input inference-only")
|
| 83 |
+
return _cuda_backend.quantize_int8_rowwise_convrot64(x.reshape(-1, self.in_features).contiguous(), self.group_size)
|
| 84 |
+
|
| 85 |
+
def forward_quantized(self, q, scale):
|
| 86 |
+
if self.training or torch.is_grad_enabled():
|
| 87 |
+
raise ValueError("CLEF ConvRot linears are inference-only")
|
| 88 |
+
if self.backend == "cute":
|
| 89 |
+
output = scaled_int8_mm(q, self.weight, scale, self.weight_scale, self.bias,
|
| 90 |
+
tile=self.tile, cluster=self.cluster)
|
| 91 |
+
elif self.backend == "torch":
|
| 92 |
+
padding = max(24, (q.shape[0] + 7) // 8 * 8) - q.shape[0]
|
| 93 |
+
qp = torch.nn.functional.pad(q, (0, 0, 0, padding)) if padding else q
|
| 94 |
+
acc = torch._int_mm(qp, self.weight.t())[:q.shape[0]]
|
| 95 |
+
output = (acc.float() * scale * self.weight_scale + self.bias.float()).to(torch.bfloat16)
|
| 96 |
+
else:
|
| 97 |
+
raise ValueError("Unknown CLEF ConvRot backend")
|
| 98 |
+
return output
|
| 99 |
+
|
| 100 |
+
def forward(self, x):
|
| 101 |
+
q, scale = self.quantize_input(x)
|
| 102 |
+
return self.forward_quantized(q, scale).view(*x.shape[:-1], self.out_features)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def shared_linear_pair(x, first, second):
|
| 106 |
+
"""Reuse a rotation only when both projections have the same quantization contract."""
|
| 107 |
+
if (isinstance(first, ConvRotLinear) and isinstance(second, ConvRotLinear)
|
| 108 |
+
and first.in_features == second.in_features and first.group_size == second.group_size
|
| 109 |
+
and first.weight.device == second.weight.device):
|
| 110 |
+
q, scale = first.quantize_input(x)
|
| 111 |
+
leading = x.shape[:-1]
|
| 112 |
+
return (first.forward_quantized(q, scale).view(*leading, first.out_features),
|
| 113 |
+
second.forward_quantized(q, scale).view(*leading, second.out_features), True)
|
| 114 |
+
return first(x), second(x), False
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def quantize_text(text, *, scope="mlp", group_size=256, backend="cute", tile=(128, 256), cluster=(1, 1),
|
| 118 |
+
calibration_index=None):
|
| 119 |
+
"""Replace text projections without retaining a second full set of weights."""
|
| 120 |
+
if (text.training or torch.is_grad_enabled() or scope not in {"mlp-gate-up", "mlp", "all"}
|
| 121 |
+
or group_size != 256 or backend not in {"cute", "torch"}):
|
| 122 |
+
raise ValueError("Quantize evaluated text weights in inference mode with scope mlp-gate-up/mlp/all")
|
| 123 |
+
names = []
|
| 124 |
+
checkpoint = None
|
| 125 |
+
if calibration_index is not None:
|
| 126 |
+
from clef_int8_calibration import CalibrationIndex
|
| 127 |
+
checkpoint = CalibrationIndex(calibration_index, text, scope)
|
| 128 |
+
for name, module in text.named_modules():
|
| 129 |
+
if not isinstance(module, torch.nn.Linear) or min(module.in_features, module.out_features) < 1024:
|
| 130 |
+
continue
|
| 131 |
+
if scope != "all" and ".mlp." not in name:
|
| 132 |
+
continue
|
| 133 |
+
if scope == "mlp-gate-up" and name.rsplit(".", 1)[-1] not in {"gate_proj", "up_proj"}:
|
| 134 |
+
continue
|
| 135 |
+
names.append(name)
|
| 136 |
+
original_bytes = quantized_bytes = 0
|
| 137 |
+
for name in names:
|
| 138 |
+
parent_name, leaf = name.rsplit(".", 1)
|
| 139 |
+
parent = text.get_submodule(parent_name)
|
| 140 |
+
linear = getattr(parent, leaf)
|
| 141 |
+
original_bytes += linear.weight.numel() * linear.weight.element_size()
|
| 142 |
+
quantized = (checkpoint.load(name, linear, backend=backend, tile=tile, cluster=cluster)
|
| 143 |
+
if checkpoint is not None else
|
| 144 |
+
ConvRotLinear(linear, group_size=group_size, backend=backend, tile=tile, cluster=cluster))
|
| 145 |
+
quantized_bytes += sum(t.numel() * t.element_size() for t in quantized.buffers())
|
| 146 |
+
setattr(parent, leaf, quantized)
|
| 147 |
+
del linear
|
| 148 |
+
torch.cuda.synchronize()
|
| 149 |
+
torch.cuda.empty_cache()
|
| 150 |
+
return {"scope": scope, "linears": len(names), "group_size": group_size,
|
| 151 |
+
"original_weight_bytes": original_bytes, "quantized_weight_bytes": quantized_bytes,
|
| 152 |
+
"backend": backend, "tile": tile, "cluster": cluster,
|
| 153 |
+
"calibration_index": str(calibration_index) if calibration_index is not None else None,
|
| 154 |
+
"total_quantized_linears": sum(isinstance(module, ConvRotLinear) for module in text.modules())}
|
runtime/clef_text_metrics.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CPU-only decision drift reporting for CLEF precision experiments."""
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
import statistics
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def decision_drift(actual, expected):
|
| 8 |
+
"""Compare nested record/question/choice logits without hiding close flips."""
|
| 9 |
+
squared_error = squared_reference = 0.0
|
| 10 |
+
max_abs = 0.0
|
| 11 |
+
decisions = []
|
| 12 |
+
for record, (candidate, reference) in enumerate(zip(actual, expected, strict=True)):
|
| 13 |
+
for question, (a, b) in enumerate(zip(candidate, reference, strict=True)):
|
| 14 |
+
if len(a) != len(b) or len(b) < 2:
|
| 15 |
+
raise ValueError("Each decision needs matching logits for at least two choices")
|
| 16 |
+
if not all(math.isfinite(value) for value in [*a, *b]):
|
| 17 |
+
raise ValueError("Non-finite CLEF logits")
|
| 18 |
+
order = sorted(range(len(b)), key=lambda i: b[i], reverse=True)
|
| 19 |
+
chosen = max(range(len(a)), key=lambda i: a[i])
|
| 20 |
+
errors = [x - y for x, y in zip(a, b, strict=True)]
|
| 21 |
+
squared_error += sum(value * value for value in errors)
|
| 22 |
+
squared_reference += sum(value * value for value in b)
|
| 23 |
+
error = max(abs(value) for value in errors)
|
| 24 |
+
max_abs = max(max_abs, error)
|
| 25 |
+
margin = b[order[0]] - b[order[1]]
|
| 26 |
+
decisions.append({"record": record, "question": question,
|
| 27 |
+
"reference_choice": order[0], "candidate_choice": chosen,
|
| 28 |
+
"matched": chosen == order[0], "reference_margin": margin,
|
| 29 |
+
"logits_max_abs": error,
|
| 30 |
+
"margin_exceeds_twice_error": margin > 2 * error})
|
| 31 |
+
if not decisions:
|
| 32 |
+
raise ValueError("No CLEF decisions to compare")
|
| 33 |
+
margins = [row["reference_margin"] for row in decisions]
|
| 34 |
+
matched = sum(row["matched"] for row in decisions)
|
| 35 |
+
return {"logits_max_abs": max_abs,
|
| 36 |
+
"logits_relative_l2": math.sqrt(squared_error) / max(math.sqrt(squared_reference), 1e-12),
|
| 37 |
+
"decisions_matched": matched, "decisions_total": len(decisions),
|
| 38 |
+
"decision_agreement": matched / len(decisions),
|
| 39 |
+
"reference_margin_min": min(margins), "reference_margin_median": statistics.median(margins),
|
| 40 |
+
"flips": [row for row in decisions if not row["matched"]], "decisions": decisions}
|
runtime/clef_text_profile.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Opt-in CLEF text traces; instrumentation never participates in warm timings."""
|
| 2 |
+
|
| 3 |
+
from collections import defaultdict
|
| 4 |
+
from contextlib import contextmanager
|
| 5 |
+
import statistics
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def text_stages(text, head=None):
|
| 11 |
+
yield "text", text
|
| 12 |
+
for name, module in text.named_modules():
|
| 13 |
+
parts = name.split(".")
|
| 14 |
+
if (len(parts) == 2 and parts[0] == "layers") or (
|
| 15 |
+
name and parts[-1] in {"input_layernorm", "post_attention_layernorm", "norm", "q_norm", "k_norm",
|
| 16 |
+
"mlp", "linear_attn", "self_attn", "gate_proj", "up_proj", "down_proj",
|
| 17 |
+
"q_proj", "k_proj", "v_proj", "o_proj", "in_proj_qkv", "in_proj_z",
|
| 18 |
+
"in_proj_a", "in_proj_b", "out_proj", "conv1d"}):
|
| 19 |
+
yield name, module
|
| 20 |
+
if head is not None:
|
| 21 |
+
yield "decision_head", head
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
@contextmanager
|
| 25 |
+
def stage_hooks(text, head=None, *, events=False):
|
| 26 |
+
handles, pending, pairs = [], defaultdict(list), defaultdict(list)
|
| 27 |
+
try:
|
| 28 |
+
for name, module in text_stages(text, head):
|
| 29 |
+
def pre(module, args, name=name):
|
| 30 |
+
scope = torch.profiler.record_function("clef." + name)
|
| 31 |
+
scope.__enter__()
|
| 32 |
+
torch.cuda.nvtx.range_push("clef." + name)
|
| 33 |
+
start = torch.cuda.Event(enable_timing=True) if events else None
|
| 34 |
+
if start is not None:
|
| 35 |
+
start.record()
|
| 36 |
+
pending[name].append((scope, start))
|
| 37 |
+
|
| 38 |
+
def post(module, args, output, name=name):
|
| 39 |
+
scope, start = pending[name].pop()
|
| 40 |
+
if start is not None:
|
| 41 |
+
end = torch.cuda.Event(enable_timing=True)
|
| 42 |
+
end.record()
|
| 43 |
+
pairs[name].append((start, end))
|
| 44 |
+
torch.cuda.nvtx.range_pop()
|
| 45 |
+
scope.__exit__(None, None, None)
|
| 46 |
+
|
| 47 |
+
handles.append(module.register_forward_pre_hook(pre))
|
| 48 |
+
handles.append(module.register_forward_hook(post, always_call=True))
|
| 49 |
+
yield pairs
|
| 50 |
+
finally:
|
| 51 |
+
for handle in handles:
|
| 52 |
+
handle.remove()
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def summarize_kernels(prof, iterations):
|
| 56 |
+
kernels, transfers = defaultdict(list), defaultdict(list)
|
| 57 |
+
for event in prof.events():
|
| 58 |
+
if event.device_type != torch.autograd.DeviceType.CUDA:
|
| 59 |
+
continue
|
| 60 |
+
target = transfers if event.name.lower().startswith(("memcpy", "memset")) else kernels
|
| 61 |
+
target[event.name].append(event.time_range.elapsed_us() / 1000)
|
| 62 |
+
rows = lambda source: sorted(
|
| 63 |
+
[{"name": name, "total_per_forward_ms": sum(values) / iterations,
|
| 64 |
+
"median_call_ms": statistics.median(values), "calls_per_forward": len(values) / iterations}
|
| 65 |
+
for name, values in source.items()], key=lambda row: -row["total_per_forward_ms"])
|
| 66 |
+
return {"kernels": rows(kernels), "transfers": rows(transfers),
|
| 67 |
+
"kernel_sum_ms": sum(map(sum, kernels.values())) / iterations,
|
| 68 |
+
"kernel_count_per_forward": sum(map(len, kernels.values())) / iterations}
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def profile_text(fn, text, directory, *, head=None, iterations=3, capture_nsys=False):
|
| 72 |
+
if iterations < 1:
|
| 73 |
+
raise ValueError("Profile iterations must be positive")
|
| 74 |
+
directory.mkdir(parents=True, exist_ok=True)
|
| 75 |
+
torch.cuda.synchronize()
|
| 76 |
+
if capture_nsys:
|
| 77 |
+
torch.cuda.cudart().cudaProfilerStart()
|
| 78 |
+
try:
|
| 79 |
+
with stage_hooks(text, head):
|
| 80 |
+
for _ in range(iterations):
|
| 81 |
+
fn()
|
| 82 |
+
torch.cuda.synchronize()
|
| 83 |
+
finally:
|
| 84 |
+
torch.cuda.cudart().cudaProfilerStop()
|
| 85 |
+
return {"capture": "nsys", "instrumented": True}
|
| 86 |
+
activities = [torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA]
|
| 87 |
+
kernel_trace = directory / "kernels.trace.json"
|
| 88 |
+
with torch.profiler.profile(activities=activities) as kernels:
|
| 89 |
+
for _ in range(iterations):
|
| 90 |
+
fn()
|
| 91 |
+
kernels.export_chrome_trace(str(kernel_trace))
|
| 92 |
+
stage_trace = directory / "stages.trace.json"
|
| 93 |
+
with stage_hooks(text, head, events=True) as pairs:
|
| 94 |
+
with torch.profiler.profile(activities=activities, record_shapes=True) as stages:
|
| 95 |
+
for _ in range(iterations):
|
| 96 |
+
fn()
|
| 97 |
+
torch.cuda.synchronize()
|
| 98 |
+
stages.export_chrome_trace(str(stage_trace))
|
| 99 |
+
spans = []
|
| 100 |
+
for name, events in pairs.items():
|
| 101 |
+
values = [start.elapsed_time(end) for start, end in events]
|
| 102 |
+
spans.append({"name": name, "total_per_forward_ms": sum(values) / iterations,
|
| 103 |
+
"median_call_ms": statistics.median(values), "calls_per_forward": len(values) / iterations})
|
| 104 |
+
return {**summarize_kernels(kernels, iterations), "kernel_trace": str(kernel_trace),
|
| 105 |
+
"stage_trace": str(stage_trace), "stages": sorted(spans, key=lambda row: -row["total_per_forward_ms"]),
|
| 106 |
+
"instrumented": True, "stage_spans_overlap": True, "stage_spans_include_launch_gaps": True,
|
| 107 |
+
"functional_fusions_are_attributed_to_parent_modules": True}
|
runtime/clef_vision_norm.py
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CuTeDSL residual + LayerNorm for CLEF's 1152-wide vision blocks."""
|
| 2 |
+
|
| 3 |
+
import operator
|
| 4 |
+
|
| 5 |
+
import cuda.bindings.driver as cuda
|
| 6 |
+
import cutlass
|
| 7 |
+
import cutlass.cute as cute
|
| 8 |
+
import torch
|
| 9 |
+
from cutlass.cute.runtime import make_fake_stream, make_fake_tensor
|
| 10 |
+
from cutlass.cutlass_dsl import dsl_user_op
|
| 11 |
+
from cutlass._mlir.dialects import llvm
|
| 12 |
+
from cutlass._mlir.extras import types as T
|
| 13 |
+
|
| 14 |
+
_CACHE = {}
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@dsl_user_op
|
| 18 |
+
def mul_rn(a, b, *, loc=None, ip=None):
|
| 19 |
+
return cutlass.Float32(llvm.inline_asm(T.f32(), [v.ir_value(loc=loc, ip=ip) for v in (a, b)],
|
| 20 |
+
"mul.rn.f32 $0, $1, $2;", "=f,f,f", has_side_effects=False, is_align_stack=False,
|
| 21 |
+
asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip))
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
@dsl_user_op
|
| 25 |
+
def fma_rn(a, b, c, *, loc=None, ip=None):
|
| 26 |
+
return cutlass.Float32(llvm.inline_asm(T.f32(), [v.ir_value(loc=loc, ip=ip) for v in (a, b, c)],
|
| 27 |
+
"fma.rn.f32 $0, $1, $2, $3;", "=f,f,f,f", has_side_effects=False, is_align_stack=False,
|
| 28 |
+
asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip))
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
@dsl_user_op
|
| 32 |
+
def reciprocal_rn(a, *, loc=None, ip=None):
|
| 33 |
+
return cutlass.Float32(llvm.inline_asm(T.f32(), [a.ir_value(loc=loc, ip=ip)],
|
| 34 |
+
"rcp.rn.f32 $0, $1;", "=f,f", has_side_effects=False, is_align_stack=False,
|
| 35 |
+
asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip))
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@cute.jit
|
| 39 |
+
def combine_welford(mean_b, var_b, count_b, mean_a, var_a, count_a):
|
| 40 |
+
delta = mean_b - mean_a
|
| 41 |
+
count = count_a + count_b
|
| 42 |
+
mean, var = cutlass.Float32(0), cutlass.Float32(0)
|
| 43 |
+
if count > 0:
|
| 44 |
+
coef = reciprocal_rn(count)
|
| 45 |
+
na = mul_rn(count_a, coef)
|
| 46 |
+
nb = mul_rn(count_b, coef)
|
| 47 |
+
mean = fma_rn(na, mean_a, mul_rn(nb, mean_b))
|
| 48 |
+
product = mul_rn(mul_rn(delta, delta), count_a)
|
| 49 |
+
var = fma_rn(product, nb, var_a + var_b)
|
| 50 |
+
return mean, var, count
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class WelfordResidualLayerNorm:
|
| 54 |
+
"""Match ATen's vec4 / 128-thread Welford tree before BF16 conversion."""
|
| 55 |
+
|
| 56 |
+
def __init__(self, width, eps):
|
| 57 |
+
self.width, self.eps = width, eps
|
| 58 |
+
self.iterations = (width + 511) // 512
|
| 59 |
+
|
| 60 |
+
@cute.jit
|
| 61 |
+
def __call__(self, x: cute.Tensor, skip: cute.Tensor, weight: cute.Tensor,
|
| 62 |
+
bias: cute.Tensor, residual: cute.Tensor, output: cute.Tensor,
|
| 63 |
+
stream: cuda.CUstream):
|
| 64 |
+
self.kernel(x, skip, weight, bias, residual, output).launch(
|
| 65 |
+
grid=(x.shape[0], 1, 1), block=(128, 1, 1), stream=stream)
|
| 66 |
+
|
| 67 |
+
@cute.kernel
|
| 68 |
+
def kernel(self, x: cute.Tensor, skip: cute.Tensor, weight: cute.Tensor,
|
| 69 |
+
bias: cute.Tensor, residual: cute.Tensor, output: cute.Tensor):
|
| 70 |
+
tid, _, _ = cute.arch.thread_idx()
|
| 71 |
+
row, _, _ = cute.arch.block_idx()
|
| 72 |
+
lane, warp = tid % 32, tid // 32
|
| 73 |
+
smem = cutlass.utils.SmemAllocator()
|
| 74 |
+
partial = smem.allocate_tensor(cutlass.Float32, cute.make_layout((4, 3)), byte_alignment=16)
|
| 75 |
+
values = cute.make_rmem_tensor((self.iterations * 4,), cutlass.Float32)
|
| 76 |
+
copy = cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), x.element_type, num_bits_per_copy=64)
|
| 77 |
+
mean, var, count = cutlass.Float32(0), cutlass.Float32(0), cutlass.Float32(0)
|
| 78 |
+
for step in cutlass.range_constexpr(self.iterations):
|
| 79 |
+
if tid * 4 + step * 512 < self.width:
|
| 80 |
+
base = cute.assume(tid * 4 + step * 512, divby=4)
|
| 81 |
+
offset = cute.assume(row * self.width + base, divby=4)
|
| 82 |
+
layout = cute.make_layout((4,))
|
| 83 |
+
xv = cute.make_tensor(x.iterator + offset, layout)
|
| 84 |
+
sv = cute.make_tensor(skip.iterator + offset, layout)
|
| 85 |
+
rv = cute.make_tensor(residual.iterator + offset, layout)
|
| 86 |
+
xr = cute.make_rmem_tensor((4,), x.element_type)
|
| 87 |
+
sr = cute.make_rmem_tensor((4,), x.element_type)
|
| 88 |
+
rr = cute.make_rmem_tensor((4,), x.element_type)
|
| 89 |
+
cute.copy(copy, xv, xr)
|
| 90 |
+
cute.copy(copy, sv, sr)
|
| 91 |
+
for i in cutlass.range_constexpr(4):
|
| 92 |
+
value = xr[i].to(cutlass.Float32) + sr[i].to(cutlass.Float32)
|
| 93 |
+
value = value.to(x.element_type).to(cutlass.Float32)
|
| 94 |
+
values[step * 4 + i] = value
|
| 95 |
+
rr[i] = value.to(x.element_type)
|
| 96 |
+
delta = value - mean
|
| 97 |
+
count = count + 1.0
|
| 98 |
+
mean = fma_rn(delta, reciprocal_rn(count), mean)
|
| 99 |
+
var = fma_rn(delta, value - mean, var)
|
| 100 |
+
cute.copy(copy, rr, rv)
|
| 101 |
+
for offset in cutlass.range_constexpr(5):
|
| 102 |
+
distance = 16 >> offset
|
| 103 |
+
other_mean = cute.arch.shuffle_sync_down(mean, distance)
|
| 104 |
+
other_var = cute.arch.shuffle_sync_down(var, distance)
|
| 105 |
+
other_count = cute.arch.shuffle_sync_down(count, distance)
|
| 106 |
+
mean, var, count = combine_welford(mean, var, count, other_mean, other_var, other_count)
|
| 107 |
+
if lane == 0:
|
| 108 |
+
partial[warp, 0], partial[warp, 1], partial[warp, 2] = mean, var, count
|
| 109 |
+
cute.arch.sync_threads()
|
| 110 |
+
if tid == 0:
|
| 111 |
+
m0, v0, c0 = combine_welford(partial[0, 0], partial[0, 1], partial[0, 2],
|
| 112 |
+
partial[2, 0], partial[2, 1], partial[2, 2])
|
| 113 |
+
m1, v1, c1 = combine_welford(partial[1, 0], partial[1, 1], partial[1, 2],
|
| 114 |
+
partial[3, 0], partial[3, 1], partial[3, 2])
|
| 115 |
+
mean, var, count = combine_welford(m0, v0, c0, m1, v1, c1)
|
| 116 |
+
partial[0, 0] = mean
|
| 117 |
+
partial[0, 1] = var / self.width
|
| 118 |
+
cute.arch.sync_threads()
|
| 119 |
+
mean = partial[0, 0]
|
| 120 |
+
inv = cute.math.rsqrt(partial[0, 1] + self.eps, fastmath=True)
|
| 121 |
+
for step in cutlass.range_constexpr(self.iterations):
|
| 122 |
+
if tid * 4 + step * 512 < self.width:
|
| 123 |
+
base = cute.assume(tid * 4 + step * 512, divby=4)
|
| 124 |
+
offset = cute.assume(row * self.width + base, divby=4)
|
| 125 |
+
layout = cute.make_layout((4,))
|
| 126 |
+
wv = cute.make_tensor(weight.iterator + base, layout)
|
| 127 |
+
bv = cute.make_tensor(bias.iterator + base, layout)
|
| 128 |
+
ov = cute.make_tensor(output.iterator + offset, layout)
|
| 129 |
+
wr = cute.make_rmem_tensor((4,), x.element_type)
|
| 130 |
+
br = cute.make_rmem_tensor((4,), x.element_type)
|
| 131 |
+
result = cute.make_rmem_tensor((4,), x.element_type)
|
| 132 |
+
cute.copy(copy, wv, wr)
|
| 133 |
+
cute.copy(copy, bv, br)
|
| 134 |
+
for i in cutlass.range_constexpr(4):
|
| 135 |
+
value = mul_rn(inv, values[step * 4 + i] - mean)
|
| 136 |
+
value = fma_rn(wr[i].to(cutlass.Float32), value, br[i].to(cutlass.Float32))
|
| 137 |
+
result[i] = value.to(x.element_type)
|
| 138 |
+
cute.copy(copy, result, ov)
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
class ResidualLayerNorm:
|
| 142 |
+
def __init__(self, width, eps):
|
| 143 |
+
self.width = width
|
| 144 |
+
self.eps = eps
|
| 145 |
+
|
| 146 |
+
@cute.jit
|
| 147 |
+
def __call__(self, x: cute.Tensor, skip: cute.Tensor, weight: cute.Tensor,
|
| 148 |
+
bias: cute.Tensor, residual: cute.Tensor, output: cute.Tensor,
|
| 149 |
+
stream: cuda.CUstream):
|
| 150 |
+
self.kernel(x, skip, weight, bias, residual, output).launch(
|
| 151 |
+
grid=(cute.ceil_div(x.shape[0], 4), 1, 1), block=(128, 1, 1), stream=stream)
|
| 152 |
+
|
| 153 |
+
@cute.kernel
|
| 154 |
+
def kernel(self, x: cute.Tensor, skip: cute.Tensor, weight: cute.Tensor,
|
| 155 |
+
bias: cute.Tensor, residual: cute.Tensor, output: cute.Tensor):
|
| 156 |
+
tid, _, _ = cute.arch.thread_idx()
|
| 157 |
+
block, _, _ = cute.arch.block_idx()
|
| 158 |
+
row = block * 4 + tid // 32
|
| 159 |
+
lane = tid % 32
|
| 160 |
+
if row < x.shape[0]:
|
| 161 |
+
values = cute.make_rmem_tensor((self.width // 32,), cutlass.Float32)
|
| 162 |
+
total = cutlass.Float32(0)
|
| 163 |
+
for i in cutlass.range_constexpr(self.width // 32):
|
| 164 |
+
col = lane + i * 32
|
| 165 |
+
value = (x[row, col].to(cutlass.Float32) + skip[row, col].to(cutlass.Float32))
|
| 166 |
+
value = value.to(x.element_type).to(cutlass.Float32)
|
| 167 |
+
values[i] = value
|
| 168 |
+
total = total + value
|
| 169 |
+
residual[row, col] = value.to(x.element_type)
|
| 170 |
+
mean = cute.arch.warp_reduction(total, operator.add) / self.width
|
| 171 |
+
variance = cutlass.Float32(0)
|
| 172 |
+
for i in cutlass.range_constexpr(self.width // 32):
|
| 173 |
+
centered = values[i] - mean
|
| 174 |
+
variance = variance + centered * centered
|
| 175 |
+
variance = cute.arch.warp_reduction(variance, operator.add) / self.width
|
| 176 |
+
inv = cute.math.rsqrt(variance + self.eps, fastmath=True)
|
| 177 |
+
for i in cutlass.range_constexpr(self.width // 32):
|
| 178 |
+
col = lane + i * 32
|
| 179 |
+
value = (values[i] - mean) * inv
|
| 180 |
+
value = value * weight[col].to(cutlass.Float32) + bias[col].to(cutlass.Float32)
|
| 181 |
+
output[row, col] = value.to(x.element_type)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def residual_layer_norm(x, skip, norm, *, algorithm="welford"):
|
| 185 |
+
if torch.is_grad_enabled():
|
| 186 |
+
raise RuntimeError("CLEF CuTe normalization is inference-only")
|
| 187 |
+
if algorithm not in {"welford", "two-pass"}:
|
| 188 |
+
raise ValueError("Unknown residual LayerNorm algorithm")
|
| 189 |
+
if x.ndim != 2 or x.shape != skip.shape or x.shape[1] % 32:
|
| 190 |
+
raise ValueError("Expected equal matrices with width divisible by 32")
|
| 191 |
+
if x.dtype != torch.bfloat16 or skip.dtype != x.dtype or not x.is_contiguous() or not skip.is_contiguous():
|
| 192 |
+
raise ValueError("Expected contiguous BF16 residual inputs")
|
| 193 |
+
if not x.is_cuda or skip.device != x.device:
|
| 194 |
+
raise ValueError("Residual inputs must be on the same CUDA device")
|
| 195 |
+
m, n = x.shape
|
| 196 |
+
if m == 0 or norm.weight is None or norm.bias is None:
|
| 197 |
+
raise ValueError("Expected nonempty inputs and affine LayerNorm")
|
| 198 |
+
if tuple(norm.normalized_shape) != (n,):
|
| 199 |
+
raise ValueError("LayerNorm width must match the residual inputs")
|
| 200 |
+
for parameter in (norm.weight, norm.bias):
|
| 201 |
+
if (parameter.shape != (n,) or parameter.device != x.device
|
| 202 |
+
or parameter.dtype != x.dtype or not parameter.is_contiguous()):
|
| 203 |
+
raise ValueError("Expected colocated contiguous BF16 LayerNorm parameters")
|
| 204 |
+
key = (x.device.index, m, n, norm.eps, algorithm)
|
| 205 |
+
fn = _CACHE.get(key)
|
| 206 |
+
if fn is None:
|
| 207 |
+
kwargs = {"memspace": cute.AddressSpace.gmem, "assumed_align": 16}
|
| 208 |
+
matrix = make_fake_tensor(cutlass.BFloat16, (m, n), stride=(n, 1), **kwargs)
|
| 209 |
+
vector = make_fake_tensor(cutlass.BFloat16, (n,), stride=(1,), **kwargs)
|
| 210 |
+
kernel = WelfordResidualLayerNorm if algorithm == "welford" else ResidualLayerNorm
|
| 211 |
+
fn = cute.compile(kernel(n, norm.eps), matrix, matrix, vector, vector,
|
| 212 |
+
matrix, matrix, make_fake_stream(use_tvm_ffi_env_stream=True),
|
| 213 |
+
options="--enable-tvm-ffi")
|
| 214 |
+
_CACHE[key] = fn
|
| 215 |
+
residual, output = torch.empty_like(x), torch.empty_like(x)
|
| 216 |
+
fn(x, skip, norm.weight, norm.bias, residual, output)
|
| 217 |
+
return residual, output
|
runtime/test_clef_int8_calibration.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import tempfile
|
| 3 |
+
import unittest
|
| 4 |
+
|
| 5 |
+
try:
|
| 6 |
+
import torch
|
| 7 |
+
from clef_int8_calibration import CalibrationIndex, CalibrationWriter, fit_affine
|
| 8 |
+
except ModuleNotFoundError as exc:
|
| 9 |
+
if exc.name not in {"torch", "safetensors"}:
|
| 10 |
+
raise
|
| 11 |
+
torch = None
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@unittest.skipIf(torch is None, "Calibration tests require torch and safetensors")
|
| 15 |
+
class CalibrationTests(unittest.TestCase):
|
| 16 |
+
def test_fit(self):
|
| 17 |
+
z = torch.arange(-20, 21, dtype=torch.float32)[:, None]
|
| 18 |
+
gain, bias = fit_affine(z, z * 1.02 + 0.7, affine=True, ridge=0)
|
| 19 |
+
torch.testing.assert_close(gain, torch.tensor([1.02]))
|
| 20 |
+
torch.testing.assert_close(bias, torch.tensor([0.7]))
|
| 21 |
+
gain, _ = fit_affine(z, z * 2, affine=False)
|
| 22 |
+
torch.testing.assert_close(gain, torch.tensor([1.03]))
|
| 23 |
+
|
| 24 |
+
def test_zero(self):
|
| 25 |
+
z = torch.zeros(24, 8)
|
| 26 |
+
gain, bias = fit_affine(z, torch.ones_like(z), affine=True)
|
| 27 |
+
torch.testing.assert_close(gain, torch.ones(8))
|
| 28 |
+
self.assertTrue(torch.isfinite(bias).all())
|
| 29 |
+
|
| 30 |
+
def test_invalid_fit(self):
|
| 31 |
+
with self.assertRaises(ValueError):
|
| 32 |
+
fit_affine(torch.ones(2, 3), torch.ones(3, 2), affine=True)
|
| 33 |
+
|
| 34 |
+
def make_checkpoint(self, directory):
|
| 35 |
+
text = torch.nn.Module()
|
| 36 |
+
text.proj = torch.nn.Linear(1024, 1024, bias=False).to(torch.bfloat16).eval()
|
| 37 |
+
weights = torch.nn.Module()
|
| 38 |
+
weights.register_buffer("weight", torch.zeros(1024, 1024, dtype=torch.int8))
|
| 39 |
+
weights.register_buffer("weight_scale", torch.ones(1024))
|
| 40 |
+
weights.register_buffer("bias", torch.zeros(1024, dtype=torch.bfloat16))
|
| 41 |
+
writer = CalibrationWriter(directory, {"scope": "all"}, max_bytes=600_000)
|
| 42 |
+
writer.add("proj", weights)
|
| 43 |
+
return text, weights, writer.finish()
|
| 44 |
+
|
| 45 |
+
def test_indexed_roundtrip(self):
|
| 46 |
+
with tempfile.TemporaryDirectory() as tmp:
|
| 47 |
+
text, weights, path = self.make_checkpoint(tmp)
|
| 48 |
+
index = CalibrationIndex(path, text, "all")
|
| 49 |
+
loaded = index.load("proj", text.proj, backend="torch", tile=(128, 256), cluster=(2, 1))
|
| 50 |
+
for leaf in ("weight", "weight_scale", "bias"):
|
| 51 |
+
torch.testing.assert_close(getattr(loaded, leaf), getattr(weights, leaf), atol=0, rtol=0)
|
| 52 |
+
self.assertEqual(len(index.files), 2)
|
| 53 |
+
self.assertEqual(index.metadata["total_size"], 1024 * 1024 + 1024 * 6)
|
| 54 |
+
|
| 55 |
+
def test_reject_bad_index(self):
|
| 56 |
+
with tempfile.TemporaryDirectory() as tmp:
|
| 57 |
+
text, _, path = self.make_checkpoint(tmp)
|
| 58 |
+
original = json.loads(path.read_text())
|
| 59 |
+
for mutate in (
|
| 60 |
+
lambda d: d["metadata"].update(total_size=1),
|
| 61 |
+
lambda d: d["weight_map"].pop("proj.bias"),
|
| 62 |
+
lambda d: d["weight_map"].update({"proj.bias": "../escape.safetensors"}),
|
| 63 |
+
lambda d: d["weight_map"].update({"proj.bias": ["bad"]}),
|
| 64 |
+
lambda d: d["weight_map"].update({"proj.bias": "missing.safetensors"}),
|
| 65 |
+
lambda d: d["weight_map"].update({"proj.bias": d["weight_map"]["proj.weight"]}),
|
| 66 |
+
lambda d: d.update(metadata=[]),
|
| 67 |
+
):
|
| 68 |
+
changed = json.loads(json.dumps(original))
|
| 69 |
+
mutate(changed)
|
| 70 |
+
path.write_text(json.dumps(changed))
|
| 71 |
+
with self.assertRaises(ValueError):
|
| 72 |
+
CalibrationIndex(path, text, "all")
|
| 73 |
+
path.write_text('{"metadata": {}, "metadata": {}, "weight_map": {}}')
|
| 74 |
+
with self.assertRaisesRegex(ValueError, "Duplicate"):
|
| 75 |
+
CalibrationIndex(path, text, "all")
|
| 76 |
+
path.write_text(json.dumps(original))
|
| 77 |
+
text.proj = torch.nn.Linear(1024, 2048).eval()
|
| 78 |
+
with self.assertRaisesRegex(ValueError, "shape/dtype"):
|
| 79 |
+
CalibrationIndex(path, text, "all")
|
| 80 |
+
|
| 81 |
+
def test_reject_invalid_values(self):
|
| 82 |
+
from safetensors.torch import save_file
|
| 83 |
+
|
| 84 |
+
with tempfile.TemporaryDirectory() as tmp:
|
| 85 |
+
text, weights, path = self.make_checkpoint(tmp)
|
| 86 |
+
index = CalibrationIndex(path, text, "all")
|
| 87 |
+
filename = index.weight_map["proj.weight_scale"]
|
| 88 |
+
for value in (0, float("nan")):
|
| 89 |
+
weights.weight_scale.fill_(value)
|
| 90 |
+
save_file({"proj.weight_scale": weights.weight_scale, "proj.bias": weights.bias}, index.files[filename])
|
| 91 |
+
index = CalibrationIndex(path, text, "all")
|
| 92 |
+
with self.assertRaisesRegex(ValueError, "Non-finite"):
|
| 93 |
+
index.load("proj", text.proj, backend="torch", tile=(128, 256), cluster=(2, 1))
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
if __name__ == "__main__":
|
| 97 |
+
unittest.main()
|
runtime/test_clef_text_metrics.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import unittest
|
| 2 |
+
|
| 3 |
+
from clef_text_metrics import decision_drift
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class DecisionDriftTest(unittest.TestCase):
|
| 7 |
+
def test_exact_and_tie(self):
|
| 8 |
+
logits = [[[1.0, 1.0], [-3.0, -1.0, 2.0]]]
|
| 9 |
+
result = decision_drift(logits, logits)
|
| 10 |
+
self.assertEqual(result["decision_agreement"], 1)
|
| 11 |
+
self.assertEqual(result["logits_relative_l2"], 0)
|
| 12 |
+
self.assertEqual(result["reference_margin_min"], 0)
|
| 13 |
+
self.assertFalse(result["decisions"][0]["margin_exceeds_twice_error"])
|
| 14 |
+
|
| 15 |
+
def test_close_flip_is_reported(self):
|
| 16 |
+
result = decision_drift([[[1.0, 1.2], [5.1, 1.0]]], [[[1.1, 1.0], [5.0, 1.0]]])
|
| 17 |
+
self.assertEqual(result["decisions_matched"], 1)
|
| 18 |
+
self.assertEqual(result["decisions_total"], 2)
|
| 19 |
+
self.assertEqual(result["flips"][0]["question"], 0)
|
| 20 |
+
self.assertEqual(result["flips"][0]["candidate_choice"], 1)
|
| 21 |
+
self.assertTrue(result["decisions"][1]["margin_exceeds_twice_error"])
|
| 22 |
+
self.assertAlmostEqual(result["logits_max_abs"], 0.2)
|
| 23 |
+
|
| 24 |
+
def test_zero_reference(self):
|
| 25 |
+
result = decision_drift([[[0.0, 0.0]]], [[[0.0, 0.0]]])
|
| 26 |
+
self.assertEqual(result["logits_relative_l2"], 0)
|
| 27 |
+
|
| 28 |
+
def test_invalid_shapes_and_values(self):
|
| 29 |
+
for actual, expected in [([], []), ([], [[[1.0, 2.0]]]),
|
| 30 |
+
([[[1.0]]], [[[1.0]]]), ([[[1.0, 2.0]]], [[[1.0, 2.0, 3.0]]]),
|
| 31 |
+
([[[float("nan"), 2.0]]], [[[1.0, 2.0]]]),
|
| 32 |
+
([[[1.0, 2.0]]], [[[float("inf"), 2.0]]])]:
|
| 33 |
+
with self.subTest(actual=actual, expected=expected), self.assertRaises(ValueError):
|
| 34 |
+
decision_drift(actual, expected)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
if __name__ == "__main__":
|
| 38 |
+
unittest.main()
|
runtime/test_clef_text_profile.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import importlib.util
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
from types import SimpleNamespace
|
| 4 |
+
import unittest
|
| 5 |
+
from unittest.mock import patch
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
spec = importlib.util.spec_from_file_location("profile_under_test", Path(__file__).with_name("clef_text_profile.py"))
|
| 9 |
+
profile = importlib.util.module_from_spec(spec)
|
| 10 |
+
with patch.dict("sys.modules", {"torch": SimpleNamespace(autograd=SimpleNamespace(DeviceType=SimpleNamespace(CUDA="cuda")))}):
|
| 11 |
+
spec.loader.exec_module(profile)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def event(name, milliseconds, device="cuda"):
|
| 15 |
+
return SimpleNamespace(name=name, device_type=device,
|
| 16 |
+
time_range=SimpleNamespace(elapsed_us=lambda: milliseconds * 1000))
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class KernelSummaryTest(unittest.TestCase):
|
| 20 |
+
def test_leaf_kernels_are_not_double_counted_with_cpu_scopes(self):
|
| 21 |
+
events = [event("clef.mlp", 30, "cpu"), event("gemm", 2), event("gemm", 4),
|
| 22 |
+
event("convrot", 1), event("Memcpy HtoD", 0.5), event("Memset", 0.2)]
|
| 23 |
+
result = profile.summarize_kernels(SimpleNamespace(events=lambda: events), 2)
|
| 24 |
+
self.assertEqual(result["kernel_sum_ms"], 3.5)
|
| 25 |
+
self.assertEqual(result["kernel_count_per_forward"], 1.5)
|
| 26 |
+
self.assertEqual(result["kernels"][0]["name"], "gemm")
|
| 27 |
+
self.assertEqual(result["kernels"][0]["total_per_forward_ms"], 3)
|
| 28 |
+
self.assertEqual(result["kernels"][0]["median_call_ms"], 3)
|
| 29 |
+
self.assertEqual(len(result["transfers"]), 2)
|
| 30 |
+
|
| 31 |
+
def test_empty_capture(self):
|
| 32 |
+
result = profile.summarize_kernels(SimpleNamespace(events=lambda: []), 3)
|
| 33 |
+
self.assertEqual(result["kernel_sum_ms"], 0)
|
| 34 |
+
self.assertEqual(result["kernels"], [])
|
| 35 |
+
|
| 36 |
+
def test_reject_zero_iterations_before_touching_gpu(self):
|
| 37 |
+
with self.assertRaises(ValueError):
|
| 38 |
+
profile.profile_text(None, None, None, iterations=0)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
if __name__ == "__main__":
|
| 42 |
+
unittest.main()
|