import{s as as,n as ns,o as is}from"../chunks/scheduler.25b97de1.js";import{S as ps,i as cs,g as p,s as a,r as T,A as os,h as c,f as t,c as n,j as es,u as x,x as F,k as ts,y as rs,a as l,v as w,d as q,t as _,w as $}from"../chunks/index.d9030fc9.js";import{C as V}from"../chunks/CodeBlock.e6cd0d95.js";import{H as ls,E as ms}from"../chunks/EditOnGithub.91d95064.js";function Ms(D){let i,E,U,v,o,C,r,L=`Half-Quadratic Quantization (HQQ) implements on-the-fly quantization via fast robust optimization. It doesn’t require calibration data and can be used to quantize any model.<br/>
Please refer to the <a href="https://github.com/mobiusml/hqq/">official package</a> for more details.`,k,m,P="For installation, we recommend you use the following approach to get the latest version and build its corresponding CUDA kernels:",B,M,Q,f,S='To quantize a model, you need to create an <a href="/docs/transformers/pr_31198/en/main_classes/quantization#transformers.HqqConfig">HqqConfig</a>. There are two ways of doing it:',H,u,N,h,Z,d,O="The second approach is especially interesting for quantizing Mixture-of-Experts (MoEs) because the experts are less affected by lower quantization settings.",I,j,K="Then you simply quantize the model as follows",W,b,A,g,G,J,ss=`HQQ supports various backends, including pure Pytorch and custom dequantization CUDA kernels. These backends are suitable for older gpus and peft/QLoRA training.
For faster inference, HQQ supports 4-bit fused kernels (TorchAO and Marlin), reaching up to 200 tokens/sec on a single 4090.
For more details on how to use the backends, please refer to <a href="https://github.com/mobiusml/hqq/?tab=readme-ov-file#backend" rel="nofollow">https://github.com/mobiusml/hqq/?tab=readme-ov-file#backend</a>`,R,y,X,z,Y;return o=new ls({props:{title:"HQQ",local:"hqq",headingTag:"h1"}}),M=new V({props:{code:"cGlwJTIwaW5zdGFsbCUyMGhxcQ==",highlighted:'pip <span class="hljs-keyword">install</span> hqq',wrap:!1}}),u=new V({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEhxcUNvbmZpZyUwQSUwQSUyMyUyME1ldGhvZCUyMDElM0ElMjBhbGwlMjBsaW5lYXIlMjBsYXllcnMlMjB3aWxsJTIwdXNlJTIwdGhlJTIwc2FtZSUyMHF1YW50aXphdGlvbiUyMGNvbmZpZyUwQXF1YW50X2NvbmZpZyUyMCUyMCUzRCUyMEhxcUNvbmZpZyhuYml0cyUzRDglMkMlMjBncm91cF9zaXplJTNENjQlMkMlMjBxdWFudF96ZXJvJTNERmFsc2UlMkMlMjBxdWFudF9zY2FsZSUzREZhbHNlJTJDJTIwYXhpcyUzRDApJTIwJTIzYXhpcyUzRDAlMjBpcyUyMHVzZWQlMjBieSUyMGRlZmF1bHQ=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, HqqConfig

<span class="hljs-comment"># Method 1: all linear layers will use the same quantization config</span>
quant_config  = HqqConfig(nbits=<span class="hljs-number">8</span>, group_size=<span class="hljs-number">64</span>, quant_zero=<span class="hljs-literal">False</span>, quant_scale=<span class="hljs-literal">False</span>, axis=<span class="hljs-number">0</span>) <span class="hljs-comment">#axis=0 is used by default</span>`,wrap:!1}}),h=new V({props:{code:"JTIzJTIwTWV0aG9kJTIwMiUzQSUyMGVhY2glMjBsaW5lYXIlMjBsYXllciUyMHdpdGglMjB0aGUlMjBzYW1lJTIwdGFnJTIwd2lsbCUyMHVzZSUyMGElMjBkZWRpY2F0ZWQlMjBxdWFudGl6YXRpb24lMjBjb25maWclMEFxNF9jb25maWclMjAlM0QlMjAlN0InbmJpdHMnJTNBNCUyQyUyMCdncm91cF9zaXplJyUzQTY0JTJDJTIwJ3F1YW50X3plcm8nJTNBRmFsc2UlMkMlMjAncXVhbnRfc2NhbGUnJTNBRmFsc2UlN0QlMEFxM19jb25maWclMjAlM0QlMjAlN0InbmJpdHMnJTNBMyUyQyUyMCdncm91cF9zaXplJyUzQTMyJTJDJTIwJ3F1YW50X3plcm8nJTNBRmFsc2UlMkMlMjAncXVhbnRfc2NhbGUnJTNBRmFsc2UlN0QlMEFxdWFudF9jb25maWclMjAlMjAlM0QlMjBIcXFDb25maWcoZHluYW1pY19jb25maWclM0QlN0IlMEElMjAlMjAnc2VsZl9hdHRuLnFfcHJvaiclM0FxNF9jb25maWclMkMlMEElMjAlMjAnc2VsZl9hdHRuLmtfcHJvaiclM0FxNF9jb25maWclMkMlMEElMjAlMjAnc2VsZl9hdHRuLnZfcHJvaiclM0FxNF9jb25maWclMkMlMEElMjAlMjAnc2VsZl9hdHRuLm9fcHJvaiclM0FxNF9jb25maWclMkMlMEElMEElMjAlMjAnbWxwLmdhdGVfcHJvaiclM0FxM19jb25maWclMkMlMEElMjAlMjAnbWxwLnVwX3Byb2onJTIwJTIwJTNBcTNfY29uZmlnJTJDJTBBJTIwJTIwJ21scC5kb3duX3Byb2onJTNBcTNfY29uZmlnJTJDJTBBJTdEKQ==",highlighted:`<span class="hljs-comment"># Method 2: each linear layer with the same tag will use a dedicated quantization config</span>
q4_config = {<span class="hljs-string">&#x27;nbits&#x27;</span>:<span class="hljs-number">4</span>, <span class="hljs-string">&#x27;group_size&#x27;</span>:<span class="hljs-number">64</span>, <span class="hljs-string">&#x27;quant_zero&#x27;</span>:<span class="hljs-literal">False</span>, <span class="hljs-string">&#x27;quant_scale&#x27;</span>:<span class="hljs-literal">False</span>}
q3_config = {<span class="hljs-string">&#x27;nbits&#x27;</span>:<span class="hljs-number">3</span>, <span class="hljs-string">&#x27;group_size&#x27;</span>:<span class="hljs-number">32</span>, <span class="hljs-string">&#x27;quant_zero&#x27;</span>:<span class="hljs-literal">False</span>, <span class="hljs-string">&#x27;quant_scale&#x27;</span>:<span class="hljs-literal">False</span>}
quant_config  = HqqConfig(dynamic_config={
  <span class="hljs-string">&#x27;self_attn.q_proj&#x27;</span>:q4_config,
  <span class="hljs-string">&#x27;self_attn.k_proj&#x27;</span>:q4_config,
  <span class="hljs-string">&#x27;self_attn.v_proj&#x27;</span>:q4_config,
  <span class="hljs-string">&#x27;self_attn.o_proj&#x27;</span>:q4_config,

  <span class="hljs-string">&#x27;mlp.gate_proj&#x27;</span>:q3_config,
  <span class="hljs-string">&#x27;mlp.up_proj&#x27;</span>  :q3_config,
  <span class="hljs-string">&#x27;mlp.down_proj&#x27;</span>:q3_config,
})`,wrap:!1}}),b=new V({props:{code:"bW9kZWwlMjAlM0QlMjB0cmFuc2Zvcm1lcnMuQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX2lkJTJDJTIwJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTIwJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmN1ZGElMjIlMkMlMjAlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTBBKQ==",highlighted:`model = transformers.AutoModelForCausalLM.from_pretrained(
    model_id, 
    torch_dtype=torch.float16, 
    device_map=<span class="hljs-string">&quot;cuda&quot;</span>, 
    quantization_config=quant_config
)`,wrap:!1}}),g=new ls({props:{title:"Optimized Runtime",local:"optimized-runtime",headingTag:"h2"}}),y=new ms({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/hqq.md"}}),{c(){i=p("meta"),E=a(),U=p("p"),v=a(),T(o.$$.fragment),C=a(),r=p("p"),r.innerHTML=L,k=a(),m=p("p"),m.textContent=P,B=a(),T(M.$$.fragment),Q=a(),f=p("p"),f.innerHTML=S,H=a(),T(u.$$.fragment),N=a(),T(h.$$.fragment),Z=a(),d=p("p"),d.textContent=O,I=a(),j=p("p"),j.textContent=K,W=a(),T(b.$$.fragment),A=a(),T(g.$$.fragment),G=a(),J=p("p"),J.innerHTML=ss,R=a(),T(y.$$.fragment),X=a(),z=p("p"),this.h()},l(s){const e=os("svelte-u9bgzb",document.head);i=c(e,"META",{name:!0,content:!0}),e.forEach(t),E=n(s),U=c(s,"P",{}),es(U).forEach(t),v=n(s),x(o.$$.fragment,s),C=n(s),r=c(s,"P",{"data-svelte-h":!0}),F(r)!=="svelte-1fxt6gs"&&(r.innerHTML=L),k=n(s),m=c(s,"P",{"data-svelte-h":!0}),F(m)!=="svelte-d1mcqh"&&(m.textContent=P),B=n(s),x(M.$$.fragment,s),Q=n(s),f=c(s,"P",{"data-svelte-h":!0}),F(f)!=="svelte-dbccqe"&&(f.innerHTML=S),H=n(s),x(u.$$.fragment,s),N=n(s),x(h.$$.fragment,s),Z=n(s),d=c(s,"P",{"data-svelte-h":!0}),F(d)!=="svelte-1domfft"&&(d.textContent=O),I=n(s),j=c(s,"P",{"data-svelte-h":!0}),F(j)!=="svelte-1gckzen"&&(j.textContent=K),W=n(s),x(b.$$.fragment,s),A=n(s),x(g.$$.fragment,s),G=n(s),J=c(s,"P",{"data-svelte-h":!0}),F(J)!=="svelte-1w4zqho"&&(J.innerHTML=ss),R=n(s),x(y.$$.fragment,s),X=n(s),z=c(s,"P",{}),es(z).forEach(t),this.h()},h(){ts(i,"name","hf:doc:metadata"),ts(i,"content",fs)},m(s,e){rs(document.head,i),l(s,E,e),l(s,U,e),l(s,v,e),w(o,s,e),l(s,C,e),l(s,r,e),l(s,k,e),l(s,m,e),l(s,B,e),w(M,s,e),l(s,Q,e),l(s,f,e),l(s,H,e),w(u,s,e),l(s,N,e),w(h,s,e),l(s,Z,e),l(s,d,e),l(s,I,e),l(s,j,e),l(s,W,e),w(b,s,e),l(s,A,e),w(g,s,e),l(s,G,e),l(s,J,e),l(s,R,e),w(y,s,e),l(s,X,e),l(s,z,e),Y=!0},p:ns,i(s){Y||(q(o.$$.fragment,s),q(M.$$.fragment,s),q(u.$$.fragment,s),q(h.$$.fragment,s),q(b.$$.fragment,s),q(g.$$.fragment,s),q(y.$$.fragment,s),Y=!0)},o(s){_(o.$$.fragment,s),_(M.$$.fragment,s),_(u.$$.fragment,s),_(h.$$.fragment,s),_(b.$$.fragment,s),_(g.$$.fragment,s),_(y.$$.fragment,s),Y=!1},d(s){s&&(t(E),t(U),t(v),t(C),t(r),t(k),t(m),t(B),t(Q),t(f),t(H),t(N),t(Z),t(d),t(I),t(j),t(W),t(A),t(G),t(J),t(R),t(X),t(z)),t(i),$(o,s),$(M,s),$(u,s),$(h,s),$(b,s),$(g,s),$(y,s)}}}const fs='{"title":"HQQ","local":"hqq","sections":[{"title":"Optimized Runtime","local":"optimized-runtime","sections":[],"depth":2}],"depth":1}';function us(D){return is(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class gs extends ps{constructor(i){super(),cs(this,i,us,Ms,as,{})}}export{gs as component};
