import{s as rt,o as dt,n as pt}from"../chunks/scheduler.25b97de1.js";import{S as ft,i as mt,g as r,s,r as x,A as ut,h as d,f as n,c as o,j as st,u as v,x as f,k as ot,y as ht,a,v as A,d as _,t as k,w as Q}from"../chunks/index.d9030fc9.js";import{T as ct}from"../chunks/Tip.baa67368.js";import{C as it}from"../chunks/CodeBlock.e6cd0d95.js";import{H as V,E as Mt}from"../chunks/EditOnGithub.91d95064.js";function $t(H){let l,m='Try AQLM on <a href="https://colab.research.google.com/drive/1-xZmBRXT5Fm3Ghn4Mwa2KRypORXb855X?usp=sharing" rel="nofollow">Google Colab</a>!';return{c(){l=r("p"),l.innerHTML=m},l(i){l=d(i,"P",{"data-svelte-h":!0}),f(l)!=="svelte-1rh1txr"&&(l.innerHTML=m)},m(i,q){a(i,l,q)},p:pt,d(i){i&&n(l)}}}function bt(H){let l,m,i,q,u,R,p,P,h,Y='Additive Quantization of Language Models (<a href="https://arxiv.org/abs/2401.06118" rel="nofollow">AQLM</a>) is a Large Language Models compression method. It quantizes multiple weights together and take advantage of interdependencies between them. AQLM represents groups of 8-16 weights as a sum of multiple vector codes.',F,c,D="Inference support for AQLM is realised in the <code>aqlm</code> library. Make sure to install it to run the models (note aqlm works only with python&gt;=3.10):",G,M,J,$,O="The library provides efficient kernels for both GPU and CPU inference and training.",X,b,tt='The instructions on how to quantize models yourself, as well as all the relevant code can be found in the corresponding GitHub <a href="https://github.com/Vahe1994/AQLM" rel="nofollow">repository</a>. To run AQLM models simply load a model that has been quantized with AQLM:',z,g,j,y,N,U,et='Starting with version <code>aqlm 1.0.2</code>, AQLM supports Parameter-Efficient Fine-Tuning in a form of <a href="https://huggingface.co/docs/peft/package_reference/lora" rel="nofollow">LoRA</a> integrated into the <a href="https://huggingface.co/blog/peft" rel="nofollow">PEFT</a> library.',S,T,B,w,nt="AQLM quantization setups vary mainly on the number of codebooks used as well as codebook sizes in bits. The most popular setups, as well as inference kernels they support are:",Z,C,at="<thead><tr><th>Kernel</th> <th>Number of codebooks</th> <th>Codebook size, bits</th> <th>Notation</th> <th>Accuracy</th> <th>Speedup</th> <th>Fast GPU inference</th> <th>Fast CPU inference</th></tr></thead> <tbody><tr><td>Triton</td> <td>K</td> <td>N</td> <td>KxN</td> <td>-</td> <td>Up to ~0.7x</td> <td>✅</td> <td>❌</td></tr> <tr><td>CUDA</td> <td>1</td> <td>16</td> <td>1x16</td> <td>Best</td> <td>Up to ~1.3x</td> <td>✅</td> <td>❌</td></tr> <tr><td>CUDA</td> <td>2</td> <td>8</td> <td>2x8</td> <td>OK</td> <td>Up to ~3.0x</td> <td>✅</td> <td>❌</td></tr> <tr><td>Numba</td> <td>K</td> <td>8</td> <td>Kx8</td> <td>Good</td> <td>Up to ~4.0x</td> <td>❌</td> <td>✅</td></tr></tbody>",W,L,I,E,K;return u=new V({props:{title:"AQLM",local:"aqlm",headingTag:"h1"}}),p=new ct({props:{warning:!1,$$slots:{default:[$t]},$$scope:{ctx:H}}}),M=new it({props:{code:"cGlwJTIwaW5zdGFsbCUyMGFxbG0lNUJncHUlMkNjcHUlNUQ=",highlighted:"pip install aqlm[gpu,cpu]",wrap:!1}}),g=new it({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQSUwQXF1YW50aXplZF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJJU1RBLURBU0xhYiUyRk1peHRyYWwtOHg3Yi1BUUxNLTJCaXQtMXgxNi1oZiUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEJTIyYXV0byUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTBBKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMklTVEEtREFTTGFiJTJGTWl4dHJhbC04eDdiLUFRTE0tMkJpdC0xeDE2LWhmJTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

quantized_model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;ISTA-DASLab/Mixtral-8x7b-AQLM-2Bit-1x16-hf&quot;</span>,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>, 
    device_map=<span class="hljs-string">&quot;auto&quot;</span>
)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;ISTA-DASLab/Mixtral-8x7b-AQLM-2Bit-1x16-hf&quot;</span>)`,wrap:!1}}),y=new V({props:{title:"PEFT",local:"peft",headingTag:"h2"}}),T=new V({props:{title:"AQLM configurations",local:"aqlm-configurations",headingTag:"h2"}}),L=new Mt({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/aqlm.md"}}),{c(){l=r("meta"),m=s(),i=r("p"),q=s(),x(u.$$.fragment),R=s(),x(p.$$.fragment),P=s(),h=r("p"),h.innerHTML=Y,F=s(),c=r("p"),c.innerHTML=D,G=s(),x(M.$$.fragment),J=s(),$=r("p"),$.textContent=O,X=s(),b=r("p"),b.innerHTML=tt,z=s(),x(g.$$.fragment),j=s(),x(y.$$.fragment),N=s(),U=r("p"),U.innerHTML=et,S=s(),x(T.$$.fragment),B=s(),w=r("p"),w.textContent=nt,Z=s(),C=r("table"),C.innerHTML=at,W=s(),x(L.$$.fragment),I=s(),E=r("p"),this.h()},l(t){const e=ut("svelte-u9bgzb",document.head);l=d(e,"META",{name:!0,content:!0}),e.forEach(n),m=o(t),i=d(t,"P",{}),st(i).forEach(n),q=o(t),v(u.$$.fragment,t),R=o(t),v(p.$$.fragment,t),P=o(t),h=d(t,"P",{"data-svelte-h":!0}),f(h)!=="svelte-11fbd25"&&(h.innerHTML=Y),F=o(t),c=d(t,"P",{"data-svelte-h":!0}),f(c)!=="svelte-1rdkef8"&&(c.innerHTML=D),G=o(t),v(M.$$.fragment,t),J=o(t),$=d(t,"P",{"data-svelte-h":!0}),f($)!=="svelte-6sf1ac"&&($.textContent=O),X=o(t),b=d(t,"P",{"data-svelte-h":!0}),f(b)!=="svelte-1urbdfd"&&(b.innerHTML=tt),z=o(t),v(g.$$.fragment,t),j=o(t),v(y.$$.fragment,t),N=o(t),U=d(t,"P",{"data-svelte-h":!0}),f(U)!=="svelte-1uurbap"&&(U.innerHTML=et),S=o(t),v(T.$$.fragment,t),B=o(t),w=d(t,"P",{"data-svelte-h":!0}),f(w)!=="svelte-1ouechy"&&(w.textContent=nt),Z=o(t),C=d(t,"TABLE",{"data-svelte-h":!0}),f(C)!=="svelte-mgp7ro"&&(C.innerHTML=at),W=o(t),v(L.$$.fragment,t),I=o(t),E=d(t,"P",{}),st(E).forEach(n),this.h()},h(){ot(l,"name","hf:doc:metadata"),ot(l,"content",gt)},m(t,e){ht(document.head,l),a(t,m,e),a(t,i,e),a(t,q,e),A(u,t,e),a(t,R,e),A(p,t,e),a(t,P,e),a(t,h,e),a(t,F,e),a(t,c,e),a(t,G,e),A(M,t,e),a(t,J,e),a(t,$,e),a(t,X,e),a(t,b,e),a(t,z,e),A(g,t,e),a(t,j,e),A(y,t,e),a(t,N,e),a(t,U,e),a(t,S,e),A(T,t,e),a(t,B,e),a(t,w,e),a(t,Z,e),a(t,C,e),a(t,W,e),A(L,t,e),a(t,I,e),a(t,E,e),K=!0},p(t,[e]){const lt={};e&2&&(lt.$$scope={dirty:e,ctx:t}),p.$set(lt)},i(t){K||(_(u.$$.fragment,t),_(p.$$.fragment,t),_(M.$$.fragment,t),_(g.$$.fragment,t),_(y.$$.fragment,t),_(T.$$.fragment,t),_(L.$$.fragment,t),K=!0)},o(t){k(u.$$.fragment,t),k(p.$$.fragment,t),k(M.$$.fragment,t),k(g.$$.fragment,t),k(y.$$.fragment,t),k(T.$$.fragment,t),k(L.$$.fragment,t),K=!1},d(t){t&&(n(m),n(i),n(q),n(R),n(P),n(h),n(F),n(c),n(G),n(J),n($),n(X),n(b),n(z),n(j),n(N),n(U),n(S),n(B),n(w),n(Z),n(C),n(W),n(I),n(E)),n(l),Q(u,t),Q(p,t),Q(M,t),Q(g,t),Q(y,t),Q(T,t),Q(L,t)}}}const gt='{"title":"AQLM","local":"aqlm","sections":[{"title":"PEFT","local":"peft","sections":[],"depth":2},{"title":"AQLM configurations","local":"aqlm-configurations","sections":[],"depth":2}],"depth":1}';function yt(H){return dt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class xt extends ft{constructor(l){super(),mt(this,l,yt,bt,rt,{})}}export{xt as component};
