import{s as te,n as ne,o as de}from"../chunks/scheduler.25b97de1.js";import{S as oe,i as ae,g as i,s as o,r as J,A as ie,h as c,f as n,c as a,j as K,u as Q,x as l,k as V,y as ce,a as d,v as W,d as X,t as Z,w as ee}from"../chunks/index.d9030fc9.js";import{H as re,E as le}from"../chunks/EditOnGithub.91d95064.js";function se(I){let r,k,v,w,s,z,h,R="Batched inputs are often different lengths, so they can’t be converted to fixed-size tensors. Padding and truncation are strategies for dealing with this problem, to create rectangular tensors from batches of varying lengths. Padding adds a special <strong>padding token</strong> to ensure shorter sequences will have the same length as either the longest sequence in a batch or the maximum length accepted by the model. Truncation works in the other direction by truncating long sequences.",L,u,N="In most cases, padding your batch to the length of the longest sequence and truncating to the maximum length a model can accept works pretty well. However, the API supports more strategies if you need them. The three arguments you need to are: <code>padding</code>, <code>truncation</code> and <code>max_length</code>.",H,m,Y="The <code>padding</code> argument controls padding. It can be a boolean or a string:",M,p,U=`<li><code>True</code> or <code>&#39;longest&#39;</code>: pad to the longest sequence in the batch (no padding is applied if you only provide
a single sequence).</li> <li><code>&#39;max_length&#39;</code>: pad to a length specified by the <code>max_length</code> argument or the maximum length accepted
by the model if no <code>max_length</code> is provided (<code>max_length=None</code>). Padding will still be applied if you only provide a single sequence.</li> <li><code>False</code> or <code>&#39;do_not_pad&#39;</code>: no padding is applied. This is the default behavior.</li>`,q,g,B="The <code>truncation</code> argument controls truncation. It can be a boolean or a string:",P,f,F=`<li><code>True</code> or <code>&#39;longest_first&#39;</code>: truncate to a maximum length specified by the <code>max_length</code> argument or
the maximum length accepted by the model if no <code>max_length</code> is provided (<code>max_length=None</code>). This will
truncate token by token, removing a token from the longest sequence in the pair until the proper length is
reached.</li> <li><code>&#39;only_second&#39;</code>: truncate to a maximum length specified by the <code>max_length</code> argument or the maximum
length accepted by the model if no <code>max_length</code> is provided (<code>max_length=None</code>). This will only truncate
the second sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.</li> <li><code>&#39;only_first&#39;</code>: truncate to a maximum length specified by the <code>max_length</code> argument or the maximum
length accepted by the model if no <code>max_length</code> is provided (<code>max_length=None</code>). This will only truncate
the first sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.</li> <li><code>False</code> or <code>&#39;do_not_truncate&#39;</code>: no truncation is applied. This is the default behavior.</li>`,$,_,j="The <code>max_length</code> argument controls the length of the padding and truncation. It can be an integer or <code>None</code>, in which case it will default to the maximum length the model can accept. If the model has no specific maximum input length, truncation or padding to <code>max_length</code> is deactivated.",E,b,O=`The following table summarizes the recommended way to setup padding and truncation. If you use pairs of input sequences in any of the following examples, you can replace <code>truncation=True</code> by a <code>STRATEGY</code> selected in
<code>[&#39;only_first&#39;, &#39;only_second&#39;, &#39;longest_first&#39;]</code>, i.e. <code>truncation=&#39;only_second&#39;</code> or <code>truncation=&#39;longest_first&#39;</code> to control how both sequences in the pair are truncated as detailed before.`,A,x,D="<thead><tr><th>Truncation</th> <th>Padding</th> <th>Instruction</th></tr></thead> <tbody><tr><td>no truncation</td> <td>no padding</td> <td><code>tokenizer(batch_sentences)</code></td></tr> <tr><td></td> <td>padding to max sequence in batch</td> <td><code>tokenizer(batch_sentences, padding=True)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, padding=&#39;longest&#39;)</code></td></tr> <tr><td></td> <td>padding to max model input length</td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;)</code></td></tr> <tr><td></td> <td>padding to specific length</td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;, max_length=42)</code></td></tr> <tr><td></td> <td>padding to a multiple of a value</td> <td><code>tokenizer(batch_sentences, padding=True, pad_to_multiple_of=8)</code></td></tr> <tr><td>truncation to max model input length</td> <td>no padding</td> <td><code>tokenizer(batch_sentences, truncation=True)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, truncation=STRATEGY)</code></td></tr> <tr><td></td> <td>padding to max sequence in batch</td> <td><code>tokenizer(batch_sentences, padding=True, truncation=True)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, padding=True, truncation=STRATEGY)</code></td></tr> <tr><td></td> <td>padding to max model input length</td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;, truncation=True)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;, truncation=STRATEGY)</code></td></tr> <tr><td></td> <td>padding to specific length</td> <td>Not possible</td></tr> <tr><td>truncation to specific length</td> <td>no padding</td> <td><code>tokenizer(batch_sentences, truncation=True, max_length=42)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, truncation=STRATEGY, max_length=42)</code></td></tr> <tr><td></td> <td>padding to max sequence in batch</td> <td><code>tokenizer(batch_sentences, padding=True, truncation=True, max_length=42)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, padding=True, truncation=STRATEGY, max_length=42)</code></td></tr> <tr><td></td> <td>padding to max model input length</td> <td>Not possible</td></tr> <tr><td></td> <td>padding to specific length</td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;, truncation=True, max_length=42)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;, truncation=STRATEGY, max_length=42)</code></td></tr></tbody>",C,T,S,y,G;return s=new re({props:{title:"Padding and truncation",local:"padding-and-truncation",headingTag:"h1"}}),T=new le({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/pad_truncation.md"}}),{c(){r=i("meta"),k=o(),v=i("p"),w=o(),J(s.$$.fragment),z=o(),h=i("p"),h.innerHTML=R,L=o(),u=i("p"),u.innerHTML=N,H=o(),m=i("p"),m.innerHTML=Y,M=o(),p=i("ul"),p.innerHTML=U,q=o(),g=i("p"),g.innerHTML=B,P=o(),f=i("ul"),f.innerHTML=F,$=o(),_=i("p"),_.innerHTML=j,E=o(),b=i("p"),b.innerHTML=O,A=o(),x=i("table"),x.innerHTML=D,C=o(),J(T.$$.fragment),S=o(),y=i("p"),this.h()},l(e){const t=ie("svelte-u9bgzb",document.head);r=c(t,"META",{name:!0,content:!0}),t.forEach(n),k=a(e),v=c(e,"P",{}),K(v).forEach(n),w=a(e),Q(s.$$.fragment,e),z=a(e),h=c(e,"P",{"data-svelte-h":!0}),l(h)!=="svelte-18l3nu8"&&(h.innerHTML=R),L=a(e),u=c(e,"P",{"data-svelte-h":!0}),l(u)!=="svelte-4dmk3p"&&(u.innerHTML=N),H=a(e),m=c(e,"P",{"data-svelte-h":!0}),l(m)!=="svelte-1xxctal"&&(m.innerHTML=Y),M=a(e),p=c(e,"UL",{"data-svelte-h":!0}),l(p)!=="svelte-1pqvfv8"&&(p.innerHTML=U),q=a(e),g=c(e,"P",{"data-svelte-h":!0}),l(g)!=="svelte-14kdvi5"&&(g.innerHTML=B),P=a(e),f=c(e,"UL",{"data-svelte-h":!0}),l(f)!=="svelte-1eic0m4"&&(f.innerHTML=F),$=a(e),_=c(e,"P",{"data-svelte-h":!0}),l(_)!=="svelte-n95phd"&&(_.innerHTML=j),E=a(e),b=c(e,"P",{"data-svelte-h":!0}),l(b)!=="svelte-1rrzmkp"&&(b.innerHTML=O),A=a(e),x=c(e,"TABLE",{"data-svelte-h":!0}),l(x)!=="svelte-1n5b2vw"&&(x.innerHTML=D),C=a(e),Q(T.$$.fragment,e),S=a(e),y=c(e,"P",{}),K(y).forEach(n),this.h()},h(){V(r,"name","hf:doc:metadata"),V(r,"content",he)},m(e,t){ce(document.head,r),d(e,k,t),d(e,v,t),d(e,w,t),W(s,e,t),d(e,z,t),d(e,h,t),d(e,L,t),d(e,u,t),d(e,H,t),d(e,m,t),d(e,M,t),d(e,p,t),d(e,q,t),d(e,g,t),d(e,P,t),d(e,f,t),d(e,$,t),d(e,_,t),d(e,E,t),d(e,b,t),d(e,A,t),d(e,x,t),d(e,C,t),W(T,e,t),d(e,S,t),d(e,y,t),G=!0},p:ne,i(e){G||(X(s.$$.fragment,e),X(T.$$.fragment,e),G=!0)},o(e){Z(s.$$.fragment,e),Z(T.$$.fragment,e),G=!1},d(e){e&&(n(k),n(v),n(w),n(z),n(h),n(L),n(u),n(H),n(m),n(M),n(p),n(q),n(g),n(P),n(f),n($),n(_),n(E),n(b),n(A),n(x),n(C),n(S),n(y)),n(r),ee(s,e),ee(T,e)}}}const he='{"title":"Padding and truncation","local":"padding-and-truncation","sections":[],"depth":1}';function ue(I){return de(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class fe extends oe{constructor(r){super(),ae(this,r,ue,se,te,{})}}export{fe as component};
