import{s as ae,n as ie,o as ne}from"../chunks/scheduler.8c3d61f6.js";import{S as se,i as oe,g as s,s as i,r as _,A as pe,h as o,f as l,c as n,j as O,u as Z,x as J,k as ee,y as re,a,v as U,d as j,t as F,w as v}from"../chunks/index.da70eac4.js";import{C as te}from"../chunks/CodeBlock.00a903b3.js";import{H as le,E as me}from"../chunks/EditOnGithub.1e64e623.js";function ue(V){let p,$,b,k,r,E,m,q='The GGUF file format is typically used to store models for inference with <a href="https://github.com/ggerganov/ggml" rel="nofollow">GGML</a> and supports a variety of block wise quantization options. Diffusers supports loading checkpoints prequantized and saved in the GGUF format via <code>from_single_file</code> loading with Model classes. Loading GGUF checkpoints via Pipelines is currently not supported.',x,u,Y='The following example will load the <a href="https://huggingface.co/black-forest-labs/FLUX.1-dev" rel="nofollow">FLUX.1 DEV</a> transformer model using the GGUF Q2_K quantization variant.',B,d,P="Before starting please install gguf in your environment",C,f,Q,c,S='Since GGUF is a single file format, use <code>~FromSingleFileMixin.from_single_file</code> to load the model and pass in the <a href="/docs/diffusers/pr_10140/en/api/quantization#diffusers.GGUFQuantizationConfig">GGUFQuantizationConfig</a>.',W,h,N="When using GGUF checkpoints, the quantized weights remain in a low memory <code>dtype</code>(typically <code>torch.uint8</code>) and are dynamically dequantized and cast to the configured <code>compute_dtype</code> during each module’s forward pass through the model. The <code>GGUFQuantizationConfig</code> allows you to set the <code>compute_dtype</code>.",z,M,D='The functions used for dynamic dequantizatation are based on the great work done by <a href="https://github.com/city96/ComfyUI-GGUF" rel="nofollow">city96</a>, who created the Pytorch ports of the original <a href="https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/gguf/quants.py" rel="nofollow"><code>numpy</code></a> implementation by <a href="https://github.com/compilade" rel="nofollow">compilade</a>.',I,g,H,y,R,w,K="<li>BF16</li> <li>Q4_0</li> <li>Q4_1</li> <li>Q5_0</li> <li>Q5_1</li> <li>Q8_0</li> <li>Q2_K</li> <li>Q3_K</li> <li>Q4_K</li> <li>Q5_K</li> <li>Q6_K</li>",X,T,L,G,A;return r=new le({props:{title:"GGUF",local:"gguf",headingTag:"h1"}}),f=new te({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1VJTIwZ2d1Zg==",highlighted:"pip install -U gguf",wrap:!1}}),g=new te({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwRmx1eFBpcGVsaW5lJTJDJTIwRmx1eFRyYW5zZm9ybWVyMkRNb2RlbCUyQyUyMEdHVUZRdWFudGl6YXRpb25Db25maWclMEElMEFja3B0X3BhdGglMjAlM0QlMjAoJTBBJTIwJTIwJTIwJTIwJTIyaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmNpdHk5NiUyRkZMVVguMS1kZXYtZ2d1ZiUyRmJsb2IlMkZtYWluJTJGZmx1eDEtZGV2LVEyX0suZ2d1ZiUyMiUwQSklMEF0cmFuc2Zvcm1lciUyMCUzRCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9zaW5nbGVfZmlsZSglMEElMjAlMjAlMjAlMjBja3B0X3BhdGglMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNER0dVRlF1YW50aXphdGlvbkNvbmZpZyhjb21wdXRlX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYpJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSklMEFwaXBlJTIwJTNEJTIwRmx1eFBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjB0cmFuc2Zvcm1lciUzRHRyYW5zZm9ybWVyJTJDJTBBJTIwJTIwJTIwJTIwZ2VuZXJhdG9yJTNEdG9yY2gubWFudWFsX3NlZWQoMCklMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTBBKSUwQXBpcGUuZW5hYmxlX21vZGVsX2NwdV9vZmZsb2FkKCklMEFwcm9tcHQlMjAlM0QlMjAlMjJBJTIwY2F0JTIwaG9sZGluZyUyMGElMjBzaWduJTIwdGhhdCUyMHNheXMlMjBoZWxsbyUyMHdvcmxkJTIyJTBBaW1hZ2UlMjAlM0QlMjBwaXBlKHByb21wdCkuaW1hZ2VzJTVCMCU1RCUwQWltYWdlLnNhdmUoJTIyZmx1eC1nZ3VmLnBuZyUyMik=",highlighted:`<span class="hljs-keyword">import</span> torch

<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline, FluxTransformer2DModel, GGUFQuantizationConfig

ckpt_path = (
    <span class="hljs-string">&quot;https://huggingface.co/city96/FLUX.1-dev-gguf/blob/main/flux1-dev-Q2_K.gguf&quot;</span>
)
transformer = FluxTransformer2DModel.from_single_file(
    ckpt_path,
    quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
    torch_dtype=torch.bfloat16,
)
pipe = FluxPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    transformer=transformer,
    generator=torch.manual_seed(<span class="hljs-number">0</span>),
    torch_dtype=torch.bfloat16,
)
pipe.enable_model_cpu_offload()
prompt = <span class="hljs-string">&quot;A cat holding a sign that says hello world&quot;</span>
image = pipe(prompt).images[<span class="hljs-number">0</span>]
image.save(<span class="hljs-string">&quot;flux-gguf.png&quot;</span>)`,wrap:!1}}),y=new le({props:{title:"Supported Quantization Types",local:"supported-quantization-types",headingTag:"h2"}}),T=new me({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/quantization/gguf.md"}}),{c(){p=s("meta"),$=i(),b=s("p"),k=i(),_(r.$$.fragment),E=i(),m=s("p"),m.innerHTML=q,x=i(),u=s("p"),u.innerHTML=Y,B=i(),d=s("p"),d.textContent=P,C=i(),_(f.$$.fragment),Q=i(),c=s("p"),c.innerHTML=S,W=i(),h=s("p"),h.innerHTML=N,z=i(),M=s("p"),M.innerHTML=D,I=i(),_(g.$$.fragment),H=i(),_(y.$$.fragment),R=i(),w=s("ul"),w.innerHTML=K,X=i(),_(T.$$.fragment),L=i(),G=s("p"),this.h()},l(e){const t=pe("svelte-u9bgzb",document.head);p=o(t,"META",{name:!0,content:!0}),t.forEach(l),$=n(e),b=o(e,"P",{}),O(b).forEach(l),k=n(e),Z(r.$$.fragment,e),E=n(e),m=o(e,"P",{"data-svelte-h":!0}),J(m)!=="svelte-th49fe"&&(m.innerHTML=q),x=n(e),u=o(e,"P",{"data-svelte-h":!0}),J(u)!=="svelte-15l4qp1"&&(u.innerHTML=Y),B=n(e),d=o(e,"P",{"data-svelte-h":!0}),J(d)!=="svelte-1s5awbc"&&(d.textContent=P),C=n(e),Z(f.$$.fragment,e),Q=n(e),c=o(e,"P",{"data-svelte-h":!0}),J(c)!=="svelte-11pcnyn"&&(c.innerHTML=S),W=n(e),h=o(e,"P",{"data-svelte-h":!0}),J(h)!=="svelte-1tiy01a"&&(h.innerHTML=N),z=n(e),M=o(e,"P",{"data-svelte-h":!0}),J(M)!=="svelte-2xs1wc"&&(M.innerHTML=D),I=n(e),Z(g.$$.fragment,e),H=n(e),Z(y.$$.fragment,e),R=n(e),w=o(e,"UL",{"data-svelte-h":!0}),J(w)!=="svelte-1dde1tf"&&(w.innerHTML=K),X=n(e),Z(T.$$.fragment,e),L=n(e),G=o(e,"P",{}),O(G).forEach(l),this.h()},h(){ee(p,"name","hf:doc:metadata"),ee(p,"content",de)},m(e,t){re(document.head,p),a(e,$,t),a(e,b,t),a(e,k,t),U(r,e,t),a(e,E,t),a(e,m,t),a(e,x,t),a(e,u,t),a(e,B,t),a(e,d,t),a(e,C,t),U(f,e,t),a(e,Q,t),a(e,c,t),a(e,W,t),a(e,h,t),a(e,z,t),a(e,M,t),a(e,I,t),U(g,e,t),a(e,H,t),U(y,e,t),a(e,R,t),a(e,w,t),a(e,X,t),U(T,e,t),a(e,L,t),a(e,G,t),A=!0},p:ie,i(e){A||(j(r.$$.fragment,e),j(f.$$.fragment,e),j(g.$$.fragment,e),j(y.$$.fragment,e),j(T.$$.fragment,e),A=!0)},o(e){F(r.$$.fragment,e),F(f.$$.fragment,e),F(g.$$.fragment,e),F(y.$$.fragment,e),F(T.$$.fragment,e),A=!1},d(e){e&&(l($),l(b),l(k),l(E),l(m),l(x),l(u),l(B),l(d),l(C),l(Q),l(c),l(W),l(h),l(z),l(M),l(I),l(H),l(R),l(w),l(X),l(L),l(G)),l(p),v(r,e),v(f,e),v(g,e),v(y,e),v(T,e)}}}const de='{"title":"GGUF","local":"gguf","sections":[{"title":"Supported Quantization Types","local":"supported-quantization-types","sections":[],"depth":2}],"depth":1}';function fe(V){return ne(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ye extends se{constructor(p){super(),oe(this,p,fe,ue,ae,{})}}export{ye as component};
