import{s as te,o as ee,n as Kt}from"../chunks/scheduler.5f3e6389.js";import{S as le,i as ae,e as p,s as n,c as m,h as ne,a as o,d as l,b as s,f as Pt,g as r,j as h,k as et,l as se,m as a,n as u,t as c,o as M,p as d}from"../chunks/index.373ab25c.js";import{T as Dt}from"../chunks/Tip.3e202241.js";import{C as ie,H as K}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.972976e0.js";import{C as b}from"../chunks/CodeBlock.dc13e59c.js";function pe(D){let i,T=`You can read the <a href="../../concept_guides/quantization">conceptual guide on quantization</a> to learn about quantization. It
explains the main concepts that you will be using when performing quantization with the
<a href="/docs/optimum/pr_68/en/onnxruntime/package_reference/quantization#optimum.onnxruntime.ORTQuantizer">ORTQuantizer</a>.`;return{c(){i=p("p"),i.innerHTML=T},l(y){i=o(y,"P",{"data-svelte-h":!0}),h(i)!=="svelte-wpf66b"&&(i.innerHTML=T)},m(y,j){a(y,i,j)},p:Kt,d(y){y&&l(i)}}}function oe(D){let i,T="Currently, only dynamic quantization is supported for Seq2Seq models.";return{c(){i=p("p"),i.textContent=T},l(y){i=o(y,"P",{"data-svelte-h":!0}),h(i)!=="svelte-1s2u00l"&&(i.textContent=T)},m(y,j){a(y,i,j)},p:Kt,d(y){y&&l(i)}}}function me(D){let i,T,y,j,C,lt,R,at,Z,Ft=`🤗 Optimum provides an <code>optimum.onnxruntime</code> package that enables you to apply quantization on many models hosted on
the Hugging Face Hub using the <a href="https://github.com/microsoft/onnxruntime/blob/master/onnxruntime/python/tools/quantization/README.md" rel="nofollow">ONNX Runtime</a>
quantization tool.`,nt,W,$t=`The quantization process is abstracted via the <a href="/docs/optimum/pr_68/en/onnxruntime/package_reference/configuration#optimum.onnxruntime.ORTConfig">ORTConfig</a> and
the <a href="/docs/optimum/pr_68/en/onnxruntime/package_reference/quantization#optimum.onnxruntime.ORTQuantizer">ORTQuantizer</a> classes. The former allows you to specify how quantization should be done,
while the latter effectively handles quantization.`,st,U,it,X,pt,z,Qt="The Optimum ONNX Runtime quantization tool can be used through Optimum command-line interface:",ot,I,mt,v,qt="Quantizing an ONNX model can be done as follows:",rt,G,ut,_,Bt="This quantize all the ONNX files in <code>onnx_model_location</code> with the AVX-512 instructions.",ct,F,Mt,$,Vt=`The <a href="/docs/optimum/pr_68/en/onnxruntime/package_reference/quantization#optimum.onnxruntime.ORTQuantizer">ORTQuantizer</a> class is used to quantize your ONNX model. The class can be initialized using
the <code>from_pretrained()</code> method, which supports different checkpoint formats.`,dt,Q,kt="<li>Using an already initialized <code>ORTModelForXXX</code> class.</li>",yt,q,ht,w,xt="<li>Using a local ONNX model from a directory.</li>",Tt,B,bt,V,jt,k,Nt=`The <a href="/docs/optimum/pr_68/en/onnxruntime/package_reference/quantization#optimum.onnxruntime.ORTQuantizer">ORTQuantizer</a> class can be used to quantize dynamically your ONNX model. Below you will
find an easy end-to-end example on how to quantize dynamically
<a href="https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english" rel="nofollow">distilbert-base-uncased-finetuned-sst-2-english</a>.`,Ut,x,wt,N,ft,A,At=`The <a href="/docs/optimum/pr_68/en/onnxruntime/package_reference/quantization#optimum.onnxruntime.ORTQuantizer">ORTQuantizer</a> class can be used to quantize statically your ONNX model. Below you will find
an easy end-to-end example on how to quantize statically
<a href="https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english" rel="nofollow">distilbert-base-uncased-finetuned-sst-2-english</a>.`,Jt,S,gt,E,Ct,Y,St=`The <a href="/docs/optimum/pr_68/en/onnxruntime/package_reference/quantization#optimum.onnxruntime.ORTQuantizer">ORTQuantizer</a> class currently doesn’t support multi-file models, like
<a href="/docs/optimum/pr_68/en/onnxruntime/package_reference/modeling_ort#optimum.onnxruntime.ORTModelForSeq2SeqLM">ORTModelForSeq2SeqLM</a>. If you want to quantize a Seq2Seq model, you have to quantize each
model’s component individually.`,Rt,f,Zt,H,Et="<li>Load seq2seq model as <code>ORTModelForSeq2SeqLM</code>.</li>",Wt,L,Xt,J,Yt="<li>Define Quantizer for encoder, decoder and decoder with past keys</li>",zt,O,It,g,Ht="<li>Quantize all models</li>",vt,P,Gt,tt,_t;return C=new ie({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),R=new K({props:{title:"Quantization",local:"quantization",headingTag:"h1"}}),U=new Dt({props:{$$slots:{default:[pe]},$$scope:{ctx:D}}}),X=new K({props:{title:"Quantizing a model to be used with Optimum’s CLI",local:"quantizing-a-model-to-be-used-with-optimums-cli",headingTag:"h2"}}),I=new b({props:{code:"b3B0aW11bS1jbGklMjBvbm54cnVudGltZSUyMHF1YW50aXplJTIwLS1oZWxwJTBBdXNhZ2UlM0ElMjBvcHRpbXVtLWNsaSUyMCUzQ2NvbW1hbmQlM0UlMjAlNUIlM0NhcmdzJTNFJTVEJTIwb25ueHJ1bnRpbWUlMjBxdWFudGl6ZSUyMCU1Qi1oJTVEJTIwLS1vbm54X21vZGVsJTIwT05OWF9NT0RFTCUyMC1vJTIwT1VUUFVUJTIwJTVCLS1wZXJfY2hhbm5lbCU1RCUyMCgtLWFybTY0JTIwJTdDJTIwLS1hdngyJTIwJTdDJTIwLS1hdng1MTIlMjAlN0MlMjAtLWF2eDUxMl92bm5pJTIwJTdDJTIwLS10ZW5zb3JydCUyMCU3QyUyMC1jJTIwQ09ORklHKSUwQSUwQW9wdGlvbnMlM0ElMEElMjAlMjAtaCUyQyUyMC0taGVscCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNob3clMjB0aGlzJTIwaGVscCUyMG1lc3NhZ2UlMjBhbmQlMjBleGl0JTBBJTIwJTIwLS1hcm02NCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMFF1YW50aXphdGlvbiUyMGZvciUyMHRoZSUyMEFSTTY0JTIwYXJjaGl0ZWN0dXJlLiUwQSUyMCUyMC0tYXZ4MiUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMFF1YW50aXphdGlvbiUyMHdpdGglMjBBVlgtMiUyMGluc3RydWN0aW9ucy4lMEElMjAlMjAtLWF2eDUxMiUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMFF1YW50aXphdGlvbiUyMHdpdGglMjBBVlgtNTEyJTIwaW5zdHJ1Y3Rpb25zLiUwQSUyMCUyMC0tYXZ4NTEyX3ZubmklMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBRdWFudGl6YXRpb24lMjB3aXRoJTIwQVZYLTUxMiUyMGFuZCUyMFZOTkklMjBpbnN0cnVjdGlvbnMuJTBBJTIwJTIwLS10ZW5zb3JydCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMFF1YW50aXphdGlvbiUyMGZvciUyME5WSURJQSUyMFRlbnNvclJUJTIwb3B0aW1pemVyLiUwQSUyMCUyMC1jJTIwQ09ORklHJTJDJTIwLS1jb25maWclMjBDT05GSUclMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNjBPUlRDb25maWclNjAlMjBmaWxlJTIwdG8lMjB1c2UlMjB0byUyMG9wdGltaXplJTIwdGhlJTIwbW9kZWwuJTBBJTBBUmVxdWlyZWQlMjBhcmd1bWVudHMlM0ElMEElMjAlMjAtLW9ubnhfbW9kZWwlMjBPTk5YX01PREVMJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwUGF0aCUyMHRvJTIwdGhlJTIwcmVwb3NpdG9yeSUyMHdoZXJlJTIwdGhlJTIwT05OWCUyMG1vZGVscyUyMHRvJTIwcXVhbnRpemUlMjBhcmUlMjBsb2NhdGVkLiUwQSUyMCUyMC1vJTIwT1VUUFVUJTJDJTIwLS1vdXRwdXQlMjBPVVRQVVQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBQYXRoJTIwdG8lMjB0aGUlMjBkaXJlY3RvcnklMjB3aGVyZSUyMHRvJTIwc3RvcmUlMjBnZW5lcmF0ZWQlMjBPTk5YJTIwbW9kZWwuJTBBJTBBT3B0aW9uYWwlMjBhcmd1bWVudHMlM0ElMEElMjAlMjAtLXBlcl9jaGFubmVsJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwQ29tcHV0ZSUyMHRoZSUyMHF1YW50aXphdGlvbiUyMHBhcmFtZXRlcnMlMjBvbiUyMGElMjBwZXItY2hhbm5lbCUyMGJhc2lzLiUwQQ==",highlighted:`optimum-cli onnxruntime quantize --<span class="hljs-built_in">help</span>
usage: optimum-cli &lt;<span class="hljs-built_in">command</span>&gt; [&lt;args&gt;] onnxruntime quantize [-h] --onnx_model ONNX_MODEL -o OUTPUT [--per_channel] (--arm64 | --avx2 | --avx512 | --avx512_vnni | --tensorrt | -c CONFIG)

options:
  -h, --<span class="hljs-built_in">help</span>            show this <span class="hljs-built_in">help</span> message and <span class="hljs-built_in">exit</span>
  --arm64               Quantization <span class="hljs-keyword">for</span> the ARM64 architecture.
  --avx2                Quantization with AVX-2 instructions.
  --avx512              Quantization with AVX-512 instructions.
  --avx512_vnni         Quantization with AVX-512 and VNNI instructions.
  --tensorrt            Quantization <span class="hljs-keyword">for</span> NVIDIA TensorRT optimizer.
  -c CONFIG, --config CONFIG
                        \`ORTConfig\` file to use to optimize the model.

Required arguments:
  --onnx_model ONNX_MODEL
                        Path to the repository <span class="hljs-built_in">where</span> the ONNX models to quantize are located.
  -o OUTPUT, --output OUTPUT
                        Path to the directory <span class="hljs-built_in">where</span> to store generated ONNX model.

Optional arguments:
  --per_channel         Compute the quantization parameters on a per-channel basis.
`,wrap:!1}}),G=new b({props:{code:"JTIwb3B0aW11bS1jbGklMjBvbm54cnVudGltZSUyMHF1YW50aXplJTIwLS1vbm54X21vZGVsJTIwb25ueF9tb2RlbF9sb2NhdGlvbiUyRiUyMC0tYXZ4NTEyJTIwLW8lMjBxdWFudGl6ZWRfbW9kZWwlMkY=",highlighted:" optimum-cli onnxruntime quantize --onnx_model onnx_model_location/ --avx512 -o quantized_model/",wrap:!1}}),F=new K({props:{title:"Creating an ORTQuantizer",local:"creating-an-ortquantizer",headingTag:"h2"}}),q=new b({props:{code:"ZnJvbSUyMG9wdGltdW0ub25ueHJ1bnRpbWUlMjBpbXBvcnQlMjBPUlRRdWFudGl6ZXIlMkMlMjBPUlRNb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEElMEFvcnRfbW9kZWwlMjAlM0QlMjBPUlRNb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMm9wdGltdW0lMkZkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZC1maW5ldHVuZWQtc3N0LTItZW5nbGlzaCUyMiUwQSklMEElMEFxdWFudGl6ZXIlMjAlM0QlMjBPUlRRdWFudGl6ZXIuZnJvbV9wcmV0cmFpbmVkKG9ydF9tb2RlbCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> optimum.onnxruntime <span class="hljs-keyword">import</span> ORTQuantizer, ORTModelForSequenceClassification

<span class="hljs-comment"># Loading ONNX Model from the Hub</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>ort_model = ORTModelForSequenceClassification.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;optimum/distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
<span class="hljs-meta">... </span>)

<span class="hljs-comment"># Create a quantizer from a ORTModelForXXX</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>quantizer = ORTQuantizer.from_pretrained(ort_model)`,wrap:!1}}),B=new b({props:{code:"ZnJvbSUyMG9wdGltdW0ub25ueHJ1bnRpbWUlMjBpbXBvcnQlMjBPUlRRdWFudGl6ZXIlMEElMEFxdWFudGl6ZXIlMjAlM0QlMjBPUlRRdWFudGl6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMnBhdGglMkZ0byUyRm1vZGVsJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> optimum.onnxruntime <span class="hljs-keyword">import</span> ORTQuantizer

<span class="hljs-comment"># This assumes a model.onnx exists in path/to/model</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>quantizer = ORTQuantizer.from_pretrained(<span class="hljs-string">&quot;path/to/model&quot;</span>)`,wrap:!1}}),V=new K({props:{title:"Apply Dynamic Quantization",local:"apply-dynamic-quantization",headingTag:"h2"}}),x=new b({props:{code:"ZnJvbSUyMG9wdGltdW0ub25ueHJ1bnRpbWUlMjBpbXBvcnQlMjBPUlRRdWFudGl6ZXIlMkMlMjBPUlRNb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEFmcm9tJTIwb3B0aW11bS5vbm54cnVudGltZS5jb25maWd1cmF0aW9uJTIwaW1wb3J0JTIwQXV0b1F1YW50aXphdGlvbkNvbmZpZyUwQSUwQW9ubnhfbW9kZWwlMjAlM0QlMjBPUlRNb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmRpc3RpbGJlcnQtYmFzZS11bmNhc2VkLWZpbmV0dW5lZC1zc3QtMi1lbmdsaXNoJTIyJTJDJTIwZXhwb3J0JTNEVHJ1ZSklMEElMEFxdWFudGl6ZXIlMjAlM0QlMjBPUlRRdWFudGl6ZXIuZnJvbV9wcmV0cmFpbmVkKG9ubnhfbW9kZWwpJTBBJTBBZHFjb25maWclMjAlM0QlMjBBdXRvUXVhbnRpemF0aW9uQ29uZmlnLmF2eDUxMl92bm5pKGlzX3N0YXRpYyUzREZhbHNlJTJDJTIwcGVyX2NoYW5uZWwlM0RGYWxzZSklMEElMEFtb2RlbF9xdWFudGl6ZWRfcGF0aCUyMCUzRCUyMHF1YW50aXplci5xdWFudGl6ZSglMEElMjAlMjAlMjAlMjBzYXZlX2RpciUzRCUyMnBhdGglMkZ0byUyRm91dHB1dCUyRm1vZGVsJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRGRxY29uZmlnJTJDJTBBKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> optimum.onnxruntime <span class="hljs-keyword">import</span> ORTQuantizer, ORTModelForSequenceClassification
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> optimum.onnxruntime.configuration <span class="hljs-keyword">import</span> AutoQuantizationConfig

<span class="hljs-comment"># Load PyTorch model and convert to ONNX</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>onnx_model = ORTModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>, export=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Create quantizer</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>quantizer = ORTQuantizer.from_pretrained(onnx_model)

<span class="hljs-comment"># Define the quantization strategy by creating the appropriate configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dqconfig = AutoQuantizationConfig.avx512_vnni(is_static=<span class="hljs-literal">False</span>, per_channel=<span class="hljs-literal">False</span>)

<span class="hljs-comment"># Quantize the model</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model_quantized_path = quantizer.quantize(
<span class="hljs-meta">... </span>    save_dir=<span class="hljs-string">&quot;path/to/output/model&quot;</span>,
<span class="hljs-meta">... </span>    quantization_config=dqconfig,
<span class="hljs-meta">... </span>)`,wrap:!1}}),N=new K({props:{title:"Static Quantization example",local:"static-quantization-example",headingTag:"h2"}}),S=new b({props:{code:"ZnJvbSUyMGZ1bmN0b29scyUyMGltcG9ydCUyMHBhcnRpYWwlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUwQWZyb20lMjBvcHRpbXVtLm9ubnhydW50aW1lJTIwaW1wb3J0JTIwT1JUUXVhbnRpemVyJTJDJTIwT1JUTW9kZWxGb3JTZXF1ZW5jZUNsYXNzaWZpY2F0aW9uJTBBZnJvbSUyMG9wdGltdW0ub25ueHJ1bnRpbWUuY29uZmlndXJhdGlvbiUyMGltcG9ydCUyMEF1dG9RdWFudGl6YXRpb25Db25maWclMkMlMjBBdXRvQ2FsaWJyYXRpb25Db25maWclMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMmRpc3RpbGJlcnQtYmFzZS11bmNhc2VkLWZpbmV0dW5lZC1zc3QtMi1lbmdsaXNoJTIyJTBBJTBBb25ueF9tb2RlbCUyMCUzRCUyME9SVE1vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbi5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBleHBvcnQlM0RUcnVlKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQXF1YW50aXplciUyMCUzRCUyME9SVFF1YW50aXplci5mcm9tX3ByZXRyYWluZWQob25ueF9tb2RlbCklMEFxY29uZmlnJTIwJTNEJTIwQXV0b1F1YW50aXphdGlvbkNvbmZpZy5hcm02NChpc19zdGF0aWMlM0RUcnVlJTJDJTIwcGVyX2NoYW5uZWwlM0RGYWxzZSklMEElMEFkZWYlMjBwcmVwcm9jZXNzX2ZuKGV4JTJDJTIwdG9rZW5pemVyKSUzQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHRva2VuaXplcihleCU1QiUyMnNlbnRlbmNlJTIyJTVEKSUwQSUwQWNhbGlicmF0aW9uX2RhdGFzZXQlMjAlM0QlMjBxdWFudGl6ZXIuZ2V0X2NhbGlicmF0aW9uX2RhdGFzZXQoJTBBJTIwJTIwJTIwJTIwJTIyZ2x1ZSUyMiUyQyUwQSUyMCUyMCUyMCUyMGRhdGFzZXRfY29uZmlnX25hbWUlM0QlMjJzc3QyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcHJlcHJvY2Vzc19mdW5jdGlvbiUzRHBhcnRpYWwocHJlcHJvY2Vzc19mbiUyQyUyMHRva2VuaXplciUzRHRva2VuaXplciklMkMlMEElMjAlMjAlMjAlMjBudW1fc2FtcGxlcyUzRDUwJTJDJTBBJTIwJTIwJTIwJTIwZGF0YXNldF9zcGxpdCUzRCUyMnRyYWluJTIyJTJDJTBBKSUwQSUwQWNhbGlicmF0aW9uX2NvbmZpZyUyMCUzRCUyMEF1dG9DYWxpYnJhdGlvbkNvbmZpZy5taW5tYXgoY2FsaWJyYXRpb25fZGF0YXNldCklMEElMEFyYW5nZXMlMjAlM0QlMjBxdWFudGl6ZXIuZml0KCUwQSUyMCUyMCUyMCUyMGRhdGFzZXQlM0RjYWxpYnJhdGlvbl9kYXRhc2V0JTJDJTBBJTIwJTIwJTIwJTIwY2FsaWJyYXRpb25fY29uZmlnJTNEY2FsaWJyYXRpb25fY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwb3BlcmF0b3JzX3RvX3F1YW50aXplJTNEcWNvbmZpZy5vcGVyYXRvcnNfdG9fcXVhbnRpemUlMkMlMEEpJTBBJTBBbW9kZWxfcXVhbnRpemVkX3BhdGglMjAlM0QlMjBxdWFudGl6ZXIucXVhbnRpemUoJTBBJTIwJTIwJTIwJTIwc2F2ZV9kaXIlM0QlMjJwYXRoJTJGdG8lMkZvdXRwdXQlMkZtb2RlbCUyMiUyQyUwQSUyMCUyMCUyMCUyMGNhbGlicmF0aW9uX3RlbnNvcnNfcmFuZ2UlM0RyYW5nZXMlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcWNvbmZpZyUyQyUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> functools <span class="hljs-keyword">import</span> partial
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> optimum.onnxruntime <span class="hljs-keyword">import</span> ORTQuantizer, ORTModelForSequenceClassification
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> optimum.onnxruntime.configuration <span class="hljs-keyword">import</span> AutoQuantizationConfig, AutoCalibrationConfig

<span class="hljs-meta">&gt;&gt;&gt; </span>model_id = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>

<span class="hljs-comment"># Load PyTorch model and convert to ONNX and create Quantizer and setup config</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>onnx_model = ORTModelForSequenceClassification.from_pretrained(model_id, export=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(model_id)
<span class="hljs-meta">&gt;&gt;&gt; </span>quantizer = ORTQuantizer.from_pretrained(onnx_model)
<span class="hljs-meta">&gt;&gt;&gt; </span>qconfig = AutoQuantizationConfig.arm64(is_static=<span class="hljs-literal">True</span>, per_channel=<span class="hljs-literal">False</span>)

<span class="hljs-comment"># Create the calibration dataset</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_fn</span>(<span class="hljs-params">ex, tokenizer</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenizer(ex[<span class="hljs-string">&quot;sentence&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>calibration_dataset = quantizer.get_calibration_dataset(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;glue&quot;</span>,
<span class="hljs-meta">... </span>    dataset_config_name=<span class="hljs-string">&quot;sst2&quot;</span>,
<span class="hljs-meta">... </span>    preprocess_function=partial(preprocess_fn, tokenizer=tokenizer),
<span class="hljs-meta">... </span>    num_samples=<span class="hljs-number">50</span>,
<span class="hljs-meta">... </span>    dataset_split=<span class="hljs-string">&quot;train&quot;</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-comment"># Create the calibration configuration containing the parameters related to calibration.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>calibration_config = AutoCalibrationConfig.minmax(calibration_dataset)

<span class="hljs-comment"># Perform the calibration step: computes the activations quantization ranges</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>ranges = quantizer.fit(
<span class="hljs-meta">... </span>    dataset=calibration_dataset,
<span class="hljs-meta">... </span>    calibration_config=calibration_config,
<span class="hljs-meta">... </span>    operators_to_quantize=qconfig.operators_to_quantize,
<span class="hljs-meta">... </span>)

<span class="hljs-comment"># Apply static quantization on the model</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model_quantized_path = quantizer.quantize(
<span class="hljs-meta">... </span>    save_dir=<span class="hljs-string">&quot;path/to/output/model&quot;</span>,
<span class="hljs-meta">... </span>    calibration_tensors_range=ranges,
<span class="hljs-meta">... </span>    quantization_config=qconfig,
<span class="hljs-meta">... </span>)`,wrap:!1}}),E=new K({props:{title:"Quantize Seq2Seq models",local:"quantize-seq2seq-models",headingTag:"h2"}}),f=new Dt({props:{warning:!0,$$slots:{default:[oe]},$$scope:{ctx:D}}}),L=new b({props:{code:"ZnJvbSUyMG9wdGltdW0ub25ueHJ1bnRpbWUlMjBpbXBvcnQlMjBPUlRRdWFudGl6ZXIlMkMlMjBPUlRNb2RlbEZvclNlcTJTZXFMTSUwQWZyb20lMjBvcHRpbXVtLm9ubnhydW50aW1lLmNvbmZpZ3VyYXRpb24lMjBpbXBvcnQlMjBBdXRvUXVhbnRpemF0aW9uQ29uZmlnJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJvcHRpbXVtJTJGdDUtc21hbGwlMjIlMEFvbm54X21vZGVsJTIwJTNEJTIwT1JUTW9kZWxGb3JTZXEyU2VxTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQW1vZGVsX2RpciUyMCUzRCUyMG9ubnhfbW9kZWwubW9kZWxfc2F2ZV9kaXI=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> optimum.onnxruntime <span class="hljs-keyword">import</span> ORTQuantizer, ORTModelForSeq2SeqLM
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> optimum.onnxruntime.configuration <span class="hljs-keyword">import</span> AutoQuantizationConfig

<span class="hljs-comment"># load Seq2Seq model and set model file directory</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model_id = <span class="hljs-string">&quot;optimum/t5-small&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>onnx_model = ORTModelForSeq2SeqLM.from_pretrained(model_id)
<span class="hljs-meta">&gt;&gt;&gt; </span>model_dir = onnx_model.model_save_dir`,wrap:!1}}),O=new b({props:{code:"ZW5jb2Rlcl9xdWFudGl6ZXIlMjAlM0QlMjBPUlRRdWFudGl6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2RpciUyQyUyMGZpbGVfbmFtZSUzRCUyMmVuY29kZXJfbW9kZWwub25ueCUyMiklMEElMEFkZWNvZGVyX3F1YW50aXplciUyMCUzRCUyME9SVFF1YW50aXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfZGlyJTJDJTIwZmlsZV9uYW1lJTNEJTIyZGVjb2Rlcl9tb2RlbC5vbm54JTIyKSUwQSUwQWRlY29kZXJfd3BfcXVhbnRpemVyJTIwJTNEJTIwT1JUUXVhbnRpemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9kaXIlMkMlMjBmaWxlX25hbWUlM0QlMjJkZWNvZGVyX3dpdGhfcGFzdF9tb2RlbC5vbm54JTIyKSUwQSUwQXF1YW50aXplciUyMCUzRCUyMCU1QmVuY29kZXJfcXVhbnRpemVyJTJDJTIwZGVjb2Rlcl9xdWFudGl6ZXIlMkMlMjBkZWNvZGVyX3dwX3F1YW50aXplciU1RA==",highlighted:`<span class="hljs-comment"># Create encoder quantizer</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>encoder_quantizer = ORTQuantizer.from_pretrained(model_dir, file_name=<span class="hljs-string">&quot;encoder_model.onnx&quot;</span>)

<span class="hljs-comment"># Create decoder quantizer</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder_quantizer = ORTQuantizer.from_pretrained(model_dir, file_name=<span class="hljs-string">&quot;decoder_model.onnx&quot;</span>)

<span class="hljs-comment"># Create decoder with past key values quantizer</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder_wp_quantizer = ORTQuantizer.from_pretrained(model_dir, file_name=<span class="hljs-string">&quot;decoder_with_past_model.onnx&quot;</span>)

<span class="hljs-comment"># Create Quantizer list</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>quantizer = [encoder_quantizer, decoder_quantizer, decoder_wp_quantizer]`,wrap:!1}}),P=new b({props:{code:"ZHFjb25maWclMjAlM0QlMjBBdXRvUXVhbnRpemF0aW9uQ29uZmlnLmF2eDUxMl92bm5pKGlzX3N0YXRpYyUzREZhbHNlJTJDJTIwcGVyX2NoYW5uZWwlM0RGYWxzZSklMEElMEFmb3IlMjBxJTIwaW4lMjBxdWFudGl6ZXIlM0ElMEElMjAlMjAlMjAlMjBxLnF1YW50aXplKHNhdmVfZGlyJTNEJTIyLiUyMiUyQ3F1YW50aXphdGlvbl9jb25maWclM0RkcWNvbmZpZyklMjAlMjAlMjMlMjBkb2N0ZXN0JTNBJTIwJTJCSUdOT1JFX1JFU1VMVA==",highlighted:`<span class="hljs-comment"># Define the quantization strategy by creating the appropriate configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dqconfig = AutoQuantizationConfig.avx512_vnni(is_static=<span class="hljs-literal">False</span>, per_channel=<span class="hljs-literal">False</span>)

<span class="hljs-comment"># Quantize the models individually</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> q <span class="hljs-keyword">in</span> quantizer:
<span class="hljs-meta">... </span>    q.quantize(save_dir=<span class="hljs-string">&quot;.&quot;</span>,quantization_config=dqconfig)  <span class="hljs-comment"># doctest: +IGNORE_RESULT</span>`,wrap:!1}}),{c(){i=p("meta"),T=n(),y=p("p"),j=n(),m(C.$$.fragment),lt=n(),m(R.$$.fragment),at=n(),Z=p("p"),Z.innerHTML=Ft,nt=n(),W=p("p"),W.innerHTML=$t,st=n(),m(U.$$.fragment),it=n(),m(X.$$.fragment),pt=n(),z=p("p"),z.textContent=Qt,ot=n(),m(I.$$.fragment),mt=n(),v=p("p"),v.textContent=qt,rt=n(),m(G.$$.fragment),ut=n(),_=p("p"),_.innerHTML=Bt,ct=n(),m(F.$$.fragment),Mt=n(),$=p("p"),$.innerHTML=Vt,dt=n(),Q=p("ol"),Q.innerHTML=kt,yt=n(),m(q.$$.fragment),ht=n(),w=p("ol"),w.innerHTML=xt,Tt=n(),m(B.$$.fragment),bt=n(),m(V.$$.fragment),jt=n(),k=p("p"),k.innerHTML=Nt,Ut=n(),m(x.$$.fragment),wt=n(),m(N.$$.fragment),ft=n(),A=p("p"),A.innerHTML=At,Jt=n(),m(S.$$.fragment),gt=n(),m(E.$$.fragment),Ct=n(),Y=p("p"),Y.innerHTML=St,Rt=n(),m(f.$$.fragment),Zt=n(),H=p("ol"),H.innerHTML=Et,Wt=n(),m(L.$$.fragment),Xt=n(),J=p("ol"),J.innerHTML=Yt,zt=n(),m(O.$$.fragment),It=n(),g=p("ol"),g.innerHTML=Ht,vt=n(),m(P.$$.fragment),Gt=n(),tt=p("p"),this.h()},l(t){const e=ne("svelte-u9bgzb",document.head);i=o(e,"META",{name:!0,content:!0}),e.forEach(l),T=s(t),y=o(t,"P",{}),Pt(y).forEach(l),j=s(t),r(C.$$.fragment,t),lt=s(t),r(R.$$.fragment,t),at=s(t),Z=o(t,"P",{"data-svelte-h":!0}),h(Z)!=="svelte-wazzwm"&&(Z.innerHTML=Ft),nt=s(t),W=o(t,"P",{"data-svelte-h":!0}),h(W)!=="svelte-198r714"&&(W.innerHTML=$t),st=s(t),r(U.$$.fragment,t),it=s(t),r(X.$$.fragment,t),pt=s(t),z=o(t,"P",{"data-svelte-h":!0}),h(z)!=="svelte-rew7gv"&&(z.textContent=Qt),ot=s(t),r(I.$$.fragment,t),mt=s(t),v=o(t,"P",{"data-svelte-h":!0}),h(v)!=="svelte-7tptkq"&&(v.textContent=qt),rt=s(t),r(G.$$.fragment,t),ut=s(t),_=o(t,"P",{"data-svelte-h":!0}),h(_)!=="svelte-q209i"&&(_.innerHTML=Bt),ct=s(t),r(F.$$.fragment,t),Mt=s(t),$=o(t,"P",{"data-svelte-h":!0}),h($)!=="svelte-7evghz"&&($.innerHTML=Vt),dt=s(t),Q=o(t,"OL",{"data-svelte-h":!0}),h(Q)!=="svelte-prnl7i"&&(Q.innerHTML=kt),yt=s(t),r(q.$$.fragment,t),ht=s(t),w=o(t,"OL",{start:!0,"data-svelte-h":!0}),h(w)!=="svelte-s0fljy"&&(w.innerHTML=xt),Tt=s(t),r(B.$$.fragment,t),bt=s(t),r(V.$$.fragment,t),jt=s(t),k=o(t,"P",{"data-svelte-h":!0}),h(k)!=="svelte-8gtckm"&&(k.innerHTML=Nt),Ut=s(t),r(x.$$.fragment,t),wt=s(t),r(N.$$.fragment,t),ft=s(t),A=o(t,"P",{"data-svelte-h":!0}),h(A)!=="svelte-9r6kwq"&&(A.innerHTML=At),Jt=s(t),r(S.$$.fragment,t),gt=s(t),r(E.$$.fragment,t),Ct=s(t),Y=o(t,"P",{"data-svelte-h":!0}),h(Y)!=="svelte-4ntmqd"&&(Y.innerHTML=St),Rt=s(t),r(f.$$.fragment,t),Zt=s(t),H=o(t,"OL",{"data-svelte-h":!0}),h(H)!=="svelte-o17da9"&&(H.innerHTML=Et),Wt=s(t),r(L.$$.fragment,t),Xt=s(t),J=o(t,"OL",{start:!0,"data-svelte-h":!0}),h(J)!=="svelte-vmrglm"&&(J.innerHTML=Yt),zt=s(t),r(O.$$.fragment,t),It=s(t),g=o(t,"OL",{start:!0,"data-svelte-h":!0}),h(g)!=="svelte-1lobl21"&&(g.innerHTML=Ht),vt=s(t),r(P.$$.fragment,t),Gt=s(t),tt=o(t,"P",{}),Pt(tt).forEach(l),this.h()},h(){et(i,"name","hf:doc:metadata"),et(i,"content",re),et(w,"start","2"),et(J,"start","2"),et(g,"start","3")},m(t,e){se(document.head,i),a(t,T,e),a(t,y,e),a(t,j,e),u(C,t,e),a(t,lt,e),u(R,t,e),a(t,at,e),a(t,Z,e),a(t,nt,e),a(t,W,e),a(t,st,e),u(U,t,e),a(t,it,e),u(X,t,e),a(t,pt,e),a(t,z,e),a(t,ot,e),u(I,t,e),a(t,mt,e),a(t,v,e),a(t,rt,e),u(G,t,e),a(t,ut,e),a(t,_,e),a(t,ct,e),u(F,t,e),a(t,Mt,e),a(t,$,e),a(t,dt,e),a(t,Q,e),a(t,yt,e),u(q,t,e),a(t,ht,e),a(t,w,e),a(t,Tt,e),u(B,t,e),a(t,bt,e),u(V,t,e),a(t,jt,e),a(t,k,e),a(t,Ut,e),u(x,t,e),a(t,wt,e),u(N,t,e),a(t,ft,e),a(t,A,e),a(t,Jt,e),u(S,t,e),a(t,gt,e),u(E,t,e),a(t,Ct,e),a(t,Y,e),a(t,Rt,e),u(f,t,e),a(t,Zt,e),a(t,H,e),a(t,Wt,e),u(L,t,e),a(t,Xt,e),a(t,J,e),a(t,zt,e),u(O,t,e),a(t,It,e),a(t,g,e),a(t,vt,e),u(P,t,e),a(t,Gt,e),a(t,tt,e),_t=!0},p(t,[e]){const Lt={};e&2&&(Lt.$$scope={dirty:e,ctx:t}),U.$set(Lt);const Ot={};e&2&&(Ot.$$scope={dirty:e,ctx:t}),f.$set(Ot)},i(t){_t||(c(C.$$.fragment,t),c(R.$$.fragment,t),c(U.$$.fragment,t),c(X.$$.fragment,t),c(I.$$.fragment,t),c(G.$$.fragment,t),c(F.$$.fragment,t),c(q.$$.fragment,t),c(B.$$.fragment,t),c(V.$$.fragment,t),c(x.$$.fragment,t),c(N.$$.fragment,t),c(S.$$.fragment,t),c(E.$$.fragment,t),c(f.$$.fragment,t),c(L.$$.fragment,t),c(O.$$.fragment,t),c(P.$$.fragment,t),_t=!0)},o(t){M(C.$$.fragment,t),M(R.$$.fragment,t),M(U.$$.fragment,t),M(X.$$.fragment,t),M(I.$$.fragment,t),M(G.$$.fragment,t),M(F.$$.fragment,t),M(q.$$.fragment,t),M(B.$$.fragment,t),M(V.$$.fragment,t),M(x.$$.fragment,t),M(N.$$.fragment,t),M(S.$$.fragment,t),M(E.$$.fragment,t),M(f.$$.fragment,t),M(L.$$.fragment,t),M(O.$$.fragment,t),M(P.$$.fragment,t),_t=!1},d(t){t&&(l(T),l(y),l(j),l(lt),l(at),l(Z),l(nt),l(W),l(st),l(it),l(pt),l(z),l(ot),l(mt),l(v),l(rt),l(ut),l(_),l(ct),l(Mt),l($),l(dt),l(Q),l(yt),l(ht),l(w),l(Tt),l(bt),l(jt),l(k),l(Ut),l(wt),l(ft),l(A),l(Jt),l(gt),l(Ct),l(Y),l(Rt),l(Zt),l(H),l(Wt),l(Xt),l(J),l(zt),l(It),l(g),l(vt),l(Gt),l(tt)),l(i),d(C,t),d(R,t),d(U,t),d(X,t),d(I,t),d(G,t),d(F,t),d(q,t),d(B,t),d(V,t),d(x,t),d(N,t),d(S,t),d(E,t),d(f,t),d(L,t),d(O,t),d(P,t)}}}const re='{"title":"Quantization","local":"quantization","sections":[{"title":"Quantizing a model to be used with Optimum’s CLI","local":"quantizing-a-model-to-be-used-with-optimums-cli","sections":[],"depth":2},{"title":"Creating an ORTQuantizer","local":"creating-an-ortquantizer","sections":[],"depth":2},{"title":"Apply Dynamic Quantization","local":"apply-dynamic-quantization","sections":[],"depth":2},{"title":"Static Quantization example","local":"static-quantization-example","sections":[],"depth":2},{"title":"Quantize Seq2Seq models","local":"quantize-seq2seq-models","sections":[],"depth":2}],"depth":1}';function ue(D){return ee(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Te extends le{constructor(i){super(),ae(this,i,ue,me,te,{})}}export{Te as component};
