import{s as ae,n as le,o as ie}from"../chunks/scheduler.25b97de1.js";import{S as oe,i as pe,g as o,s as a,r as j,A as me,h as p,f as s,c as l,j as te,u as q,x as _,k as se,y as re,a as n,v as x,d as C,t as $,w as T}from"../chunks/index.d9030fc9.js";import{C as S}from"../chunks/CodeBlock.e6cd0d95.js";import{H as ne,E as ce}from"../chunks/EditOnGithub.91d95064.js";function fe(V){let i,Q,J,Z,m,v,r,P=`Half-Quadratic Quantization (HQQ) implements on-the-fly quantization via fast robust optimization. It doesn’t require calibration data and can be used to quantize any model.<br/>
Please refer to the <a href="https://github.com/mobiusml/hqq/">official package</a> for more details.`,X,c,L="For installation, we recommend you use the following approach to get the latest version and build its corresponding CUDA kernels:",E,f,k,u,D='To quantize a model, you need to create an <a href="/docs/transformers/pr_22105/en/main_classes/quantization#transformers.HqqConfig">HqqConfig</a>. There are two ways of doing it:',G,M,F,d,H,y,O="The second approach is especially interesting for quantizing Mixture-of-Experts (MoEs) because the experts are less affected by lower quantization settings.",N,h,K="Then you simply quantize the model as follows",W,b,I,U,B,g,ee=`HQQ supports various backends, including pure PyTorch and custom dequantization CUDA kernels. These backends are suitable for older gpus and peft/QLoRA training.
For faster inference, HQQ supports 4-bit fused kernels (TorchAO and Marlin), reaching up to 200 tokens/sec on a single 4090.
For more details on how to use the backends, please refer to <a href="https://github.com/mobiusml/hqq/?tab=readme-ov-file#backend" rel="nofollow">https://github.com/mobiusml/hqq/?tab=readme-ov-file#backend</a>`,A,w,R,z,Y;return m=new ne({props:{title:"HQQ",local:"hqq",headingTag:"h1"}}),f=new S({props:{code:"cGlwJTIwaW5zdGFsbCUyMGhxcQ==",highlighted:'pip <span class="hljs-keyword">install</span> hqq',wrap:!1}}),M=new S({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEhxcUNvbmZpZyUwQSUwQSUyMyUyME1ldGhvZCUyMDElM0ElMjBhbGwlMjBsaW5lYXIlMjBsYXllcnMlMjB3aWxsJTIwdXNlJTIwdGhlJTIwc2FtZSUyMHF1YW50aXphdGlvbiUyMGNvbmZpZyUwQXF1YW50X2NvbmZpZyUyMCUyMCUzRCUyMEhxcUNvbmZpZyhuYml0cyUzRDglMkMlMjBncm91cF9zaXplJTNENjQp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, HqqConfig

<span class="hljs-comment"># Method 1: all linear layers will use the same quantization config</span>
quant_config  = HqqConfig(nbits=<span class="hljs-number">8</span>, group_size=<span class="hljs-number">64</span>)`,wrap:!1}}),d=new S({props:{code:"JTIzJTIwTWV0aG9kJTIwMiUzQSUyMGVhY2glMjBsaW5lYXIlMjBsYXllciUyMHdpdGglMjB0aGUlMjBzYW1lJTIwdGFnJTIwd2lsbCUyMHVzZSUyMGElMjBkZWRpY2F0ZWQlMjBxdWFudGl6YXRpb24lMjBjb25maWclMEFxNF9jb25maWclMjAlM0QlMjAlN0InbmJpdHMnJTNBNCUyQyUyMCdncm91cF9zaXplJyUzQTY0JTdEJTBBcTNfY29uZmlnJTIwJTNEJTIwJTdCJ25iaXRzJyUzQTMlMkMlMjAnZ3JvdXBfc2l6ZSclM0EzMiU3RCUwQXF1YW50X2NvbmZpZyUyMCUyMCUzRCUyMEhxcUNvbmZpZyhkeW5hbWljX2NvbmZpZyUzRCU3QiUwQSUyMCUyMCdzZWxmX2F0dG4ucV9wcm9qJyUzQXE0X2NvbmZpZyUyQyUwQSUyMCUyMCdzZWxmX2F0dG4ua19wcm9qJyUzQXE0X2NvbmZpZyUyQyUwQSUyMCUyMCdzZWxmX2F0dG4udl9wcm9qJyUzQXE0X2NvbmZpZyUyQyUwQSUyMCUyMCdzZWxmX2F0dG4ub19wcm9qJyUzQXE0X2NvbmZpZyUyQyUwQSUwQSUyMCUyMCdtbHAuZ2F0ZV9wcm9qJyUzQXEzX2NvbmZpZyUyQyUwQSUyMCUyMCdtbHAudXBfcHJvaiclMjAlMjAlM0FxM19jb25maWclMkMlMEElMjAlMjAnbWxwLmRvd25fcHJvaiclM0FxM19jb25maWclMkMlMEElN0Qp",highlighted:`<span class="hljs-comment"># Method 2: each linear layer with the same tag will use a dedicated quantization config</span>
q4_config = {<span class="hljs-string">&#x27;nbits&#x27;</span>:<span class="hljs-number">4</span>, <span class="hljs-string">&#x27;group_size&#x27;</span>:<span class="hljs-number">64</span>}
q3_config = {<span class="hljs-string">&#x27;nbits&#x27;</span>:<span class="hljs-number">3</span>, <span class="hljs-string">&#x27;group_size&#x27;</span>:<span class="hljs-number">32</span>}
quant_config  = HqqConfig(dynamic_config={
  <span class="hljs-string">&#x27;self_attn.q_proj&#x27;</span>:q4_config,
  <span class="hljs-string">&#x27;self_attn.k_proj&#x27;</span>:q4_config,
  <span class="hljs-string">&#x27;self_attn.v_proj&#x27;</span>:q4_config,
  <span class="hljs-string">&#x27;self_attn.o_proj&#x27;</span>:q4_config,

  <span class="hljs-string">&#x27;mlp.gate_proj&#x27;</span>:q3_config,
  <span class="hljs-string">&#x27;mlp.up_proj&#x27;</span>  :q3_config,
  <span class="hljs-string">&#x27;mlp.down_proj&#x27;</span>:q3_config,
})`,wrap:!1}}),b=new S({props:{code:"bW9kZWwlMjAlM0QlMjB0cmFuc2Zvcm1lcnMuQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX2lkJTJDJTIwJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTIwJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmN1ZGElMjIlMkMlMjAlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTBBKQ==",highlighted:`model = transformers.AutoModelForCausalLM.from_pretrained(
    model_id, 
    torch_dtype=torch.float16, 
    device_map=<span class="hljs-string">&quot;cuda&quot;</span>, 
    quantization_config=quant_config
)`,wrap:!1}}),U=new ne({props:{title:"Optimized Runtime",local:"optimized-runtime",headingTag:"h2"}}),w=new ce({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/hqq.md"}}),{c(){i=o("meta"),Q=a(),J=o("p"),Z=a(),j(m.$$.fragment),v=a(),r=o("p"),r.innerHTML=P,X=a(),c=o("p"),c.textContent=L,E=a(),j(f.$$.fragment),k=a(),u=o("p"),u.innerHTML=D,G=a(),j(M.$$.fragment),F=a(),j(d.$$.fragment),H=a(),y=o("p"),y.textContent=O,N=a(),h=o("p"),h.textContent=K,W=a(),j(b.$$.fragment),I=a(),j(U.$$.fragment),B=a(),g=o("p"),g.innerHTML=ee,A=a(),j(w.$$.fragment),R=a(),z=o("p"),this.h()},l(e){const t=me("svelte-u9bgzb",document.head);i=p(t,"META",{name:!0,content:!0}),t.forEach(s),Q=l(e),J=p(e,"P",{}),te(J).forEach(s),Z=l(e),q(m.$$.fragment,e),v=l(e),r=p(e,"P",{"data-svelte-h":!0}),_(r)!=="svelte-1fxt6gs"&&(r.innerHTML=P),X=l(e),c=p(e,"P",{"data-svelte-h":!0}),_(c)!=="svelte-d1mcqh"&&(c.textContent=L),E=l(e),q(f.$$.fragment,e),k=l(e),u=p(e,"P",{"data-svelte-h":!0}),_(u)!=="svelte-19ci94o"&&(u.innerHTML=D),G=l(e),q(M.$$.fragment,e),F=l(e),q(d.$$.fragment,e),H=l(e),y=p(e,"P",{"data-svelte-h":!0}),_(y)!=="svelte-1domfft"&&(y.textContent=O),N=l(e),h=p(e,"P",{"data-svelte-h":!0}),_(h)!=="svelte-1gckzen"&&(h.textContent=K),W=l(e),q(b.$$.fragment,e),I=l(e),q(U.$$.fragment,e),B=l(e),g=p(e,"P",{"data-svelte-h":!0}),_(g)!=="svelte-hjesn0"&&(g.innerHTML=ee),A=l(e),q(w.$$.fragment,e),R=l(e),z=p(e,"P",{}),te(z).forEach(s),this.h()},h(){se(i,"name","hf:doc:metadata"),se(i,"content",ue)},m(e,t){re(document.head,i),n(e,Q,t),n(e,J,t),n(e,Z,t),x(m,e,t),n(e,v,t),n(e,r,t),n(e,X,t),n(e,c,t),n(e,E,t),x(f,e,t),n(e,k,t),n(e,u,t),n(e,G,t),x(M,e,t),n(e,F,t),x(d,e,t),n(e,H,t),n(e,y,t),n(e,N,t),n(e,h,t),n(e,W,t),x(b,e,t),n(e,I,t),x(U,e,t),n(e,B,t),n(e,g,t),n(e,A,t),x(w,e,t),n(e,R,t),n(e,z,t),Y=!0},p:le,i(e){Y||(C(m.$$.fragment,e),C(f.$$.fragment,e),C(M.$$.fragment,e),C(d.$$.fragment,e),C(b.$$.fragment,e),C(U.$$.fragment,e),C(w.$$.fragment,e),Y=!0)},o(e){$(m.$$.fragment,e),$(f.$$.fragment,e),$(M.$$.fragment,e),$(d.$$.fragment,e),$(b.$$.fragment,e),$(U.$$.fragment,e),$(w.$$.fragment,e),Y=!1},d(e){e&&(s(Q),s(J),s(Z),s(v),s(r),s(X),s(c),s(E),s(k),s(u),s(G),s(F),s(H),s(y),s(N),s(h),s(W),s(I),s(B),s(g),s(A),s(R),s(z)),s(i),T(m,e),T(f,e),T(M,e),T(d,e),T(b,e),T(U,e),T(w,e)}}}const ue='{"title":"HQQ","local":"hqq","sections":[{"title":"Optimized Runtime","local":"optimized-runtime","sections":[],"depth":2}],"depth":1}';function Me(V){return ie(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ue extends oe{constructor(i){super(),pe(this,i,Me,fe,ae,{})}}export{Ue as component};
