import{s as ae,n as ie,o as ne}from"../chunks/scheduler.8c3d61f6.js";import{S as se,i as oe,g as s,s as i,r as _,A as pe,h as o,f as l,c as n,j as O,u as U,x as b,k as ee,y as re,a,v as Z,d as F,t as j,w as $}from"../chunks/index.da70eac4.js";import{C as te}from"../chunks/CodeBlock.00a903b3.js";import{H as le,E as me}from"../chunks/EditOnGithub.1e64e623.js";function fe(X){let p,v,J,k,r,C,m,q='The GGUF file format is typically used to store models for inference with <a href="https://github.com/ggerganov/ggml" rel="nofollow">GGML</a> and supports a variety of block wise quantization options. Diffusers supports loading checkpoints prequantized and saved in the GGUF format via <code>from_single_file</code> loading with Model classes. Loading GGUF checkpoints via Pipelines is currently not supported.',E,f,A='The following example will load the <a href="https://huggingface.co/black-forest-labs/FLUX.1-dev" rel="nofollow">FLUX.1 DEV</a> transformer model using the GGUF Q2_K quantization variant.',I,u,S="Before starting please install gguf in your environment",Q,c,x,d,P='Since GGUF is a single file format, use <code>~FromSingleFileMixin.from_single_file</code> to load the model and pass in the <a href="/docs/diffusers/pr_10037/en/api/quantization#diffusers.GGUFQuantizationConfig">GGUFQuantizationConfig</a>.',R,h,N="When using GGUF checkpoints, the quantized weights remain in a low memory <code>dtype</code>(typically <code>torch.uint8</code>) and are dynamically dequantized and cast to the configured <code>compute_dtype</code> during each module’s forward pass through the model. The <code>GGUFQuantizationConfig</code> allows you to set the <code>compute_dtype</code>.",z,g,D='The functions used for dynamic dequantizatation are based on the great work done by <a href="https://github.com/city96/ComfyUI-GGUF" rel="nofollow">city96</a>, who created the Pytorch ports of the original <a href="https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/gguf/quants.py" rel="nofollow"><code>numpy</code></a> implementation by <a href="https://github.com/compilade" rel="nofollow">compilade</a>.',W,M,B,y,H,w,K="<li>BF16</li> <li>Q4_0</li> <li>Q4_1</li> <li>Q5_0</li> <li>Q5_1</li> <li>Q8_0</li> <li>Q2_K</li> <li>Q3_K</li> <li>Q4_K</li> <li>Q5_K</li> <li>Q6_K</li>",L,T,V,G,Y;return r=new le({props:{title:"GGUF",local:"gguf",headingTag:"h1"}}),c=new te({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1VJTIwZ2d1Zg==",highlighted:"pip install -U gguf",wrap:!1}}),M=new te({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwRmx1eFBpcGVsaW5lJTJDJTIwRmx1eFRyYW5zZm9ybWVyMkRNb2RlbCUyQyUyMEdHVUZRdWFudGl6YXRpb25Db25maWclMEElMEFja3B0X3BhdGglMjAlM0QlMjAoJTBBJTIwJTIwJTIwJTIwJTIyaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmNpdHk5NiUyRkZMVVguMS1kZXYtZ2d1ZiUyRmJsb2IlMkZtYWluJTJGZmx1eDEtZGV2LVEyX0suZ2d1ZiUyMiUwQSklMEF0cmFuc2Zvcm1lciUyMCUzRCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9zaW5nbGVfZmlsZSglMEElMjAlMjAlMjAlMjBja3B0X3BhdGglMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNER0dVRlF1YW50aXphdGlvbkNvbmZpZyhjb21wdXRlX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYpJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSklMEFwaXBlJTIwJTNEJTIwRmx1eFBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjB0cmFuc2Zvcm1lciUzRHRyYW5zZm9ybWVyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSklMEFwaXBlLmVuYWJsZV9tb2RlbF9jcHVfb2ZmbG9hZCgpJTBBcHJvbXB0JTIwJTNEJTIwJTIyQSUyMGNhdCUyMGhvbGRpbmclMjBhJTIwc2lnbiUyMHRoYXQlMjBzYXlzJTIwaGVsbG8lMjB3b3JsZCUyMiUwQWltYWdlJTIwJTNEJTIwcGlwZShwcm9tcHQlMkMlMjBnZW5lcmF0b3IlM0R0b3JjaC5tYW51YWxfc2VlZCgwKSkuaW1hZ2VzJTVCMCU1RCUwQWltYWdlLnNhdmUoJTIyZmx1eC1nZ3VmLnBuZyUyMik=",highlighted:`<span class="hljs-keyword">import</span> torch

<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline, FluxTransformer2DModel, GGUFQuantizationConfig

ckpt_path = (
    <span class="hljs-string">&quot;https://huggingface.co/city96/FLUX.1-dev-gguf/blob/main/flux1-dev-Q2_K.gguf&quot;</span>
)
transformer = FluxTransformer2DModel.from_single_file(
    ckpt_path,
    quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
    torch_dtype=torch.bfloat16,
)
pipe = FluxPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    transformer=transformer,
    torch_dtype=torch.bfloat16,
)
pipe.enable_model_cpu_offload()
prompt = <span class="hljs-string">&quot;A cat holding a sign that says hello world&quot;</span>
image = pipe(prompt, generator=torch.manual_seed(<span class="hljs-number">0</span>)).images[<span class="hljs-number">0</span>]
image.save(<span class="hljs-string">&quot;flux-gguf.png&quot;</span>)`,wrap:!1}}),y=new le({props:{title:"Supported Quantization Types",local:"supported-quantization-types",headingTag:"h2"}}),T=new me({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/quantization/gguf.md"}}),{c(){p=s("meta"),v=i(),J=s("p"),k=i(),_(r.$$.fragment),C=i(),m=s("p"),m.innerHTML=q,E=i(),f=s("p"),f.innerHTML=A,I=i(),u=s("p"),u.textContent=S,Q=i(),_(c.$$.fragment),x=i(),d=s("p"),d.innerHTML=P,R=i(),h=s("p"),h.innerHTML=N,z=i(),g=s("p"),g.innerHTML=D,W=i(),_(M.$$.fragment),B=i(),_(y.$$.fragment),H=i(),w=s("ul"),w.innerHTML=K,L=i(),_(T.$$.fragment),V=i(),G=s("p"),this.h()},l(e){const t=pe("svelte-u9bgzb",document.head);p=o(t,"META",{name:!0,content:!0}),t.forEach(l),v=n(e),J=o(e,"P",{}),O(J).forEach(l),k=n(e),U(r.$$.fragment,e),C=n(e),m=o(e,"P",{"data-svelte-h":!0}),b(m)!=="svelte-th49fe"&&(m.innerHTML=q),E=n(e),f=o(e,"P",{"data-svelte-h":!0}),b(f)!=="svelte-15l4qp1"&&(f.innerHTML=A),I=n(e),u=o(e,"P",{"data-svelte-h":!0}),b(u)!=="svelte-1s5awbc"&&(u.textContent=S),Q=n(e),U(c.$$.fragment,e),x=n(e),d=o(e,"P",{"data-svelte-h":!0}),b(d)!=="svelte-8c2wsg"&&(d.innerHTML=P),R=n(e),h=o(e,"P",{"data-svelte-h":!0}),b(h)!=="svelte-1tiy01a"&&(h.innerHTML=N),z=n(e),g=o(e,"P",{"data-svelte-h":!0}),b(g)!=="svelte-2xs1wc"&&(g.innerHTML=D),W=n(e),U(M.$$.fragment,e),B=n(e),U(y.$$.fragment,e),H=n(e),w=o(e,"UL",{"data-svelte-h":!0}),b(w)!=="svelte-1dde1tf"&&(w.innerHTML=K),L=n(e),U(T.$$.fragment,e),V=n(e),G=o(e,"P",{}),O(G).forEach(l),this.h()},h(){ee(p,"name","hf:doc:metadata"),ee(p,"content",ue)},m(e,t){re(document.head,p),a(e,v,t),a(e,J,t),a(e,k,t),Z(r,e,t),a(e,C,t),a(e,m,t),a(e,E,t),a(e,f,t),a(e,I,t),a(e,u,t),a(e,Q,t),Z(c,e,t),a(e,x,t),a(e,d,t),a(e,R,t),a(e,h,t),a(e,z,t),a(e,g,t),a(e,W,t),Z(M,e,t),a(e,B,t),Z(y,e,t),a(e,H,t),a(e,w,t),a(e,L,t),Z(T,e,t),a(e,V,t),a(e,G,t),Y=!0},p:ie,i(e){Y||(F(r.$$.fragment,e),F(c.$$.fragment,e),F(M.$$.fragment,e),F(y.$$.fragment,e),F(T.$$.fragment,e),Y=!0)},o(e){j(r.$$.fragment,e),j(c.$$.fragment,e),j(M.$$.fragment,e),j(y.$$.fragment,e),j(T.$$.fragment,e),Y=!1},d(e){e&&(l(v),l(J),l(k),l(C),l(m),l(E),l(f),l(I),l(u),l(Q),l(x),l(d),l(R),l(h),l(z),l(g),l(W),l(B),l(H),l(w),l(L),l(V),l(G)),l(p),$(r,e),$(c,e),$(M,e),$(y,e),$(T,e)}}}const ue='{"title":"GGUF","local":"gguf","sections":[{"title":"Supported Quantization Types","local":"supported-quantization-types","sections":[],"depth":2}],"depth":1}';function ce(X){return ne(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ye extends se{constructor(p){super(),oe(this,p,ce,fe,ae,{})}}export{ye as component};
