import{s as C,n as V,o as Q}from"../chunks/scheduler.25b97de1.js";import{S as N,i as x,g as h,s,r as U,A as S,h as b,f as a,c as o,j as E,u as F,x as v,k as H,y as A,a as n,v as k,d as X,t as R,w as B}from"../chunks/index.d9030fc9.js";import{C as q}from"../chunks/CodeBlock.e6cd0d95.js";import{H as L,E as P}from"../chunks/EditOnGithub.91d95064.js";function D(z){let l,f,M,w,i,W,r,I='<a href="https://github.com/pytorch/ao" rel="nofollow">TorchAO</a> is an architecture optimization library for PyTorch, it provides high performance dtypes, optimization techniques and kernels for inference and training, featuring composability with native PyTorch features like <code>torch.compile</code>, FSDP etc.. Some benchmark numbers can be found <a href="https://github.com/pytorch/ao/tree/main?tab=readme-ov-file#without-intrusive-code-changes" rel="nofollow">here</a>',J,p,Y="Before you begin, make sure the following libraries are installed with their latest version:",T,c,g,m,j,u,$='torchao quantization is implemented with tensor subclasses, currently it does not work with huggingface serialization, both the safetensor option and <a href="https://github.com/huggingface/transformers/issues/32364" rel="nofollow">non-safetensor option</a>, we’ll update here with instructions when it’s working.',_,d,Z,y,G;return i=new L({props:{title:"TorchAO",local:"torchao",headingTag:"h1"}}),c=new q({props:{code:"cGlwJTIwaW5zdGFsbCUyMC0tdXBncmFkZSUyMHRvcmNoJTIwdG9yY2hhbw==",highlighted:"pip install --upgrade torch torchao",wrap:!1}}),m=new q({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRvcmNoQW9Db25maWclMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMEF1dG9Ub2tlbml6ZXIlMEElMEFtb2RlbF9uYW1lJTIwJTNEJTIwJTIybWV0YS1sbGFtYSUyRk1ldGEtTGxhbWEtMy04QiUyMiUwQSUyMyUyMFdlJTIwc3VwcG9ydCUyMGludDRfd2VpZ2h0X29ubHklMkMlMjBpbnQ4X3dlaWdodF9vbmx5JTIwYW5kJTIwaW50OF9keW5hbWljX2FjdGl2YXRpb25faW50OF93ZWlnaHQlMEElMjMlMjBNb3JlJTIwZXhhbXBsZXMlMjBhbmQlMjBkb2N1bWVudGF0aW9ucyUyMGZvciUyMGFyZ3VtZW50cyUyMGNhbiUyMGJlJTIwZm91bmQlMjBpbiUyMGh0dHBzJTNBJTJGJTJGZ2l0aHViLmNvbSUyRnB5dG9yY2glMkZhbyUyRnRyZWUlMkZtYWluJTJGdG9yY2hhbyUyRnF1YW50aXphdGlvbiUyM290aGVyLWF2YWlsYWJsZS1xdWFudGl6YXRpb24tdGVjaG5pcXVlcyUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBUb3JjaEFvQ29uZmlnKCUyMmludDRfd2VpZ2h0X29ubHklMjIlMkMlMjBncm91cF9zaXplJTNEMTI4KSUwQXF1YW50aXplZF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZChtb2RlbF9uYW1lJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyklMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9uYW1lKSUwQWlucHV0X3RleHQlMjAlM0QlMjAlMjJXaGF0JTIwYXJlJTIwd2UlMjBoYXZpbmclMjBmb3IlMjBkaW5uZXIlM0YlMjIlMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBJTBBJTIzJTIwY29tcGlsZSUyMHRoZSUyMHF1YW50aXpkJTIwbW9kZWwlMjB0byUyMGdldCUyMHNwZWVkdXAlMEFpbXBvcnQlMjB0b3JjaGFvJTBBdG9yY2hhby5xdWFudGl6YXRpb24udXRpbHMucmVjb21tZW5kZWRfaW5kdWN0b3JfY29uZmlnX3NldHRlcigpJTBBcXVhbnRpemVkX21vZGVsJTIwJTNEJTIwdG9yY2guY29tcGlsZShxdWFudGl6ZWRfbW9kZWwlMkMlMjBtb2RlJTNEJTIybWF4LWF1dG90dW5lJTIyKSUwQSUwQW91dHB1dCUyMCUzRCUyMHF1YW50aXplZF9tb2RlbC5nZW5lcmF0ZSgqKmlucHV0X2lkcyUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMTApJTBBcHJpbnQodG9rZW5pemVyLmRlY29kZShvdXRwdXQlNUIwJTVEJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TorchAoConfig, AutoModelForCausalLM, AutoTokenizer

model_name = <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B&quot;</span>
<span class="hljs-comment"># We support int4_weight_only, int8_weight_only and int8_dynamic_activation_int8_weight</span>
<span class="hljs-comment"># More examples and documentations for arguments can be found in https://github.com/pytorch/ao/tree/main/torchao/quantization#other-available-quantization-techniques</span>
quantization_config = TorchAoConfig(<span class="hljs-string">&quot;int4_weight_only&quot;</span>, group_size=<span class="hljs-number">128</span>)
quantized_model = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, quantization_config=quantization_config)

tokenizer = AutoTokenizer.from_pretrained(model_name)
input_text = <span class="hljs-string">&quot;What are we having for dinner?&quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-comment"># compile the quantizd model to get speedup</span>
<span class="hljs-keyword">import</span> torchao
torchao.quantization.utils.recommended_inductor_config_setter()
quantized_model = torch.<span class="hljs-built_in">compile</span>(quantized_model, mode=<span class="hljs-string">&quot;max-autotune&quot;</span>)

output = quantized_model.generate(**input_ids, max_new_tokens=<span class="hljs-number">10</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),d=new P({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/torchao.md"}}),{c(){l=h("meta"),f=s(),M=h("p"),w=s(),U(i.$$.fragment),W=s(),r=h("p"),r.innerHTML=I,J=s(),p=h("p"),p.textContent=Y,T=s(),U(c.$$.fragment),g=s(),U(m.$$.fragment),j=s(),u=h("p"),u.innerHTML=$,_=s(),U(d.$$.fragment),Z=s(),y=h("p"),this.h()},l(t){const e=S("svelte-u9bgzb",document.head);l=b(e,"META",{name:!0,content:!0}),e.forEach(a),f=o(t),M=b(t,"P",{}),E(M).forEach(a),w=o(t),F(i.$$.fragment,t),W=o(t),r=b(t,"P",{"data-svelte-h":!0}),v(r)!=="svelte-f2tpfl"&&(r.innerHTML=I),J=o(t),p=b(t,"P",{"data-svelte-h":!0}),v(p)!=="svelte-1wrsrbm"&&(p.textContent=Y),T=o(t),F(c.$$.fragment,t),g=o(t),F(m.$$.fragment,t),j=o(t),u=b(t,"P",{"data-svelte-h":!0}),v(u)!=="svelte-78510w"&&(u.innerHTML=$),_=o(t),F(d.$$.fragment,t),Z=o(t),y=b(t,"P",{}),E(y).forEach(a),this.h()},h(){H(l,"name","hf:doc:metadata"),H(l,"content",K)},m(t,e){A(document.head,l),n(t,f,e),n(t,M,e),n(t,w,e),k(i,t,e),n(t,W,e),n(t,r,e),n(t,J,e),n(t,p,e),n(t,T,e),k(c,t,e),n(t,g,e),k(m,t,e),n(t,j,e),n(t,u,e),n(t,_,e),k(d,t,e),n(t,Z,e),n(t,y,e),G=!0},p:V,i(t){G||(X(i.$$.fragment,t),X(c.$$.fragment,t),X(m.$$.fragment,t),X(d.$$.fragment,t),G=!0)},o(t){R(i.$$.fragment,t),R(c.$$.fragment,t),R(m.$$.fragment,t),R(d.$$.fragment,t),G=!1},d(t){t&&(a(f),a(M),a(w),a(W),a(r),a(J),a(p),a(T),a(g),a(j),a(u),a(_),a(Z),a(y)),a(l),B(i,t),B(c,t),B(m,t),B(d,t)}}}const K='{"title":"TorchAO","local":"torchao","sections":[],"depth":1}';function O(z){return Q(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class lt extends N{constructor(l){super(),x(this,l,O,D,C,{})}}export{lt as component};
