import{s as La,o as Aa,n as Es}from"../chunks/scheduler.25b97de1.js";import{S as Da,i as Ka,g as o,s as n,r as p,A as Pa,h as i,f as s,c as l,j as qa,u as c,x as r,k as Ys,y as Oa,a,v as d,d as m,t as h,w as g}from"../chunks/index.d9030fc9.js";import{T as Ss}from"../chunks/Tip.baa67368.js";import{C as b}from"../chunks/CodeBlock.e6cd0d95.js";import{H as M,E as en}from"../chunks/EditOnGithub.91d95064.js";function tn(j){let u,f="The API for the streamer classes is still under development and may change in the future.";return{c(){u=o("p"),u.textContent=f},l(y){u=i(y,"P",{"data-svelte-h":!0}),r(u)!=="svelte-gv2g1g"&&(u.textContent=f)},m(y,w){a(y,u,w)},p:Es,d(y){y&&s(u)}}}function sn(j){let u,f=`The WatermarkDetector internally relies on the proportion of “green” tokens, and whether generated text follows the coloring pattern.
That is why it is recommended to strip off the prompt text, if it is much longer than the generated text.
This also can have an effect when one sequence in the batch is a lot longer causing other rows to be padded.
Additionally, the detector <strong>must</strong> be initiated with identical watermark configuration arguments used when generating.`;return{c(){u=o("p"),u.innerHTML=f},l(y){u=i(y,"P",{"data-svelte-h":!0}),r(u)!=="svelte-ge8sma"&&(u.innerHTML=f)},m(y,w){a(y,u,w)},p:Es,d(y){y&&s(u)}}}function an(j){let u,f=`Selecting a given decoding strategy is not the only way you can influence the outcome of <code>generate()</code> with your model.
The decoding strategies act based (mostly) on the logits, the distribution of probabilities for the next token, and
thus selecting a good logits manipulation strategy can go a long way! In other words, manipulating the logits is another
dimension you can act upon, in addition to selecting a decoding strategy. Popular logits manipulation strategies include
<code>top_p</code>, <code>min_p</code>, and <code>repetition_penalty</code> — you can check the full list in the <a href="/docs/transformers/pr_30530/en/main_classes/text_generation#transformers.GenerationConfig">GenerationConfig</a> class.`;return{c(){u=o("p"),u.innerHTML=f},l(y){u=i(y,"P",{"data-svelte-h":!0}),r(u)!=="svelte-6g3f2n"&&(u.innerHTML=f)},m(y,w){a(y,u,w)},p:Es,d(y){y&&s(u)}}}function nn(j){let u,f,y,w,Z,ot,v,qs=`Text generation is essential to many NLP tasks, such as open-ended text generation, summarization, translation, and
more. It also plays a role in a variety of mixed-modality applications that have text as an output like speech-to-text
and vision-to-text. Some of the models that can generate text include
GPT2, XLNet, OpenAI GPT, CTRL, TransformerXL, XLM, Bart, T5, GIT, Whisper.`,it,W,Ls=`Check out a few examples that use <a href="/docs/transformers/pr_30530/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a> method to produce
text outputs for different tasks:`,rt,G,As='<li><a href="./tasks/summarization#inference">Text summarization</a></li> <li><a href="./model_doc/git#transformers.GitForCausalLM.forward.example">Image captioning</a></li> <li><a href="./model_doc/whisper#transformers.WhisperForConditionalGeneration.forward.example">Audio transcription</a></li>',pt,B,Ds=`Note that the inputs to the generate method depend on the model’s modality. They are returned by the model’s preprocessor
class, such as AutoTokenizer or AutoProcessor. If a model’s preprocessor creates more than one kind of input, pass all
the inputs to generate(). You can learn more about the individual model’s preprocessor in the corresponding model’s documentation.`,ct,X,Ks=`The process of selecting output tokens to generate text is known as decoding, and you can customize the decoding strategy
that the <code>generate()</code> method will use. Modifying a decoding strategy does not change the values of any trainable parameters.
However, it can have a noticeable impact on the quality of the generated output. It can help reduce repetition in the text
and make it more coherent.`,dt,C,Ps="This guide describes:",mt,V,Os="<li>default generation configuration</li> <li>common decoding strategies and their main parameters</li> <li>saving and sharing custom generation configurations with your fine-tuned model on 🤗 Hub</li>",ht,R,gt,_,ea=`A decoding strategy for a model is defined in its generation configuration. When using pre-trained models for inference
within a <a href="/docs/transformers/pr_30530/en/main_classes/pipelines#transformers.pipeline">pipeline()</a>, the models call the <code>PreTrainedModel.generate()</code> method that applies a default generation
configuration under the hood. The default configuration is also used when no custom configuration has been saved with
the model.`,ut,I,ta=`When you load a model explicitly, you can inspect the generation configuration that comes with it through
<code>model.generation_config</code>:`,yt,$,Mt,H,sa=`Printing out the <code>model.generation_config</code> reveals only the values that are different from the default generation
configuration, and does not list any of the default values.`,bt,N,aa=`The default generation configuration limits the size of the output combined with the input prompt to a maximum of 20
tokens to avoid running into resource limitations. The default decoding strategy is greedy search, which is the simplest decoding strategy that picks a token with the highest probability as the next token. For many tasks
and small output sizes this works well. However, when used to generate longer outputs, greedy search can start
producing highly repetitive results.`,ft,z,wt,x,na="You can override any <code>generation_config</code> by passing the parameters and their values directly to the <code>generate</code> method:",Tt,F,jt,Q,la=`Even if the default decoding strategy mostly works for your task, you can still tweak a few things. Some of the
commonly adjusted parameters include:`,Jt,Y,oa=`<li><code>max_new_tokens</code>: the maximum number of tokens to generate. In other words, the size of the output sequence, not
including the tokens in the prompt. As an alternative to using the output’s length as a stopping criteria, you can choose
to stop generation whenever the full generation exceeds some amount of time. To learn more, check <a href="/docs/transformers/pr_30530/en/internal/generation_utils#transformers.StoppingCriteria">StoppingCriteria</a>.</li> <li><code>num_beams</code>: by specifying a number of beams higher than 1, you are effectively switching from greedy search to
beam search. This strategy evaluates several hypotheses at each time step and eventually chooses the hypothesis that
has the overall highest probability for the entire sequence. This has the advantage of identifying high-probability
sequences that start with a lower probability initial tokens and would’ve been ignored by the greedy search. Visualize how it works <a href="https://huggingface.co/spaces/m-ric/beam_search_visualizer" rel="nofollow">here</a>.</li> <li><code>do_sample</code>: if set to <code>True</code>, this parameter enables decoding strategies such as multinomial sampling, beam-search
multinomial sampling, Top-K sampling and Top-p sampling. All these strategies select the next token from the probability
distribution over the entire vocabulary with various strategy-specific adjustments.</li> <li><code>num_return_sequences</code>: the number of sequence candidates to return for each input. This option is only available for
the decoding strategies that support multiple sequence candidates, e.g. variations of beam search and sampling. Decoding
strategies like greedy search and contrastive search return a single output sequence.</li>`,Ut,S,kt,E,ia="If you would like to share your fine-tuned model with a specific generation configuration, you can:",Zt,q,ra='<li>Create a <a href="/docs/transformers/pr_30530/en/main_classes/text_generation#transformers.GenerationConfig">GenerationConfig</a> class instance</li> <li>Specify the decoding strategy parameters</li> <li>Save your generation configuration with <a href="/docs/transformers/pr_30530/en/main_classes/text_generation#transformers.GenerationConfig.save_pretrained">GenerationConfig.save_pretrained()</a>, making sure to leave its <code>config_file_name</code> argument empty</li> <li>Set <code>push_to_hub</code> to <code>True</code> to upload your config to the model’s repo</li>',vt,L,Wt,A,pa=`You can also store several generation configurations in a single directory, making use of the <code>config_file_name</code>
argument in <a href="/docs/transformers/pr_30530/en/main_classes/text_generation#transformers.GenerationConfig.save_pretrained">GenerationConfig.save_pretrained()</a>. You can later instantiate them with <a href="/docs/transformers/pr_30530/en/main_classes/text_generation#transformers.GenerationConfig.from_pretrained">GenerationConfig.from_pretrained()</a>. This is useful if you want to
store several generation configurations for a single model (e.g. one for creative text generation with sampling, and
one for summarization with beam search). You must have the right Hub permissions to add configuration files to a model.`,Gt,D,Bt,K,Xt,P,ca=`The <code>generate()</code> supports streaming, through its <code>streamer</code> input. The <code>streamer</code> input is compatible with any instance
from a class that has the following methods: <code>put()</code> and <code>end()</code>. Internally, <code>put()</code> is used to push new tokens and
<code>end()</code> is used to flag the end of text generation.`,Ct,J,Vt,O,da=`In practice, you can craft your own streaming class for all sorts of purposes! We also have basic streaming classes
ready for you to use. For example, you can use the <a href="/docs/transformers/pr_30530/en/internal/generation_utils#transformers.TextStreamer">TextStreamer</a> class to stream the output of <code>generate()</code> into
your screen, one word at a time:`,Rt,ee,_t,te,It,se,ma=`The <code>generate()</code> supports watermarking the generated text by randomly marking a portion of tokens as “green”.
When generating the “green” will have a small ‘bias’ value added to their logits, thus having a higher chance to be generated.
The watermarked text can be detected by calculating the proportion of “green” tokens in the text and estimating how likely it is
statistically to obtain that amount of “green” tokens for human-generated text. This watermarking strategy was proposed in the paper
<a href="https://arxiv.org/abs/2306.04634" rel="nofollow">“On the Reliability of Watermarks for Large Language Models”</a>. For more information on
the inner functioning of watermarking, it is recommended to refer to the paper.`,$t,ae,ha=`The watermarking can be used with any generative model in <code>tranformers</code> and does not require an extra classification model
to detect watermarked text. To trigger watermarking, pass in a <a href="/docs/transformers/pr_30530/en/internal/generation_utils#transformers.WatermarkingConfig">WatermarkingConfig</a> with needed arguments directly to the
<code>.generate()</code> method or add it to the <a href="/docs/transformers/pr_30530/en/main_classes/text_generation#transformers.GenerationConfig">GenerationConfig</a>. Watermarked text can be later detected with a <a href="/docs/transformers/pr_30530/en/internal/generation_utils#transformers.WatermarkDetector">WatermarkDetector</a>.`,Ht,U,Nt,ne,ga=`Let’s generate some text with watermarking. In the below code snippet, we set the bias to 2.5 which is a value that
will be added to “green” tokens’ logits. After generating watermarked text, we can pass it directly to the <code>WatermarkDetector</code>
to check if the text is machine-generated (outputs <code>True</code> for machine-generated and <code>False</code> otherwise).`,zt,le,xt,oe,Ft,ie,ua=`Certain combinations of the <code>generate()</code> parameters, and ultimately <code>generation_config</code>, can be used to enable specific
decoding strategies. If you are new to this concept, we recommend reading
<a href="https://huggingface.co/blog/how-to-generate" rel="nofollow">this blog post that illustrates how common decoding strategies work</a>.`,Qt,re,ya="Here, we’ll show some of the parameters that control the decoding strategies and illustrate how you can use them.",Yt,k,St,pe,Et,ce,Ma="<code>generate</code> uses greedy search decoding by default so you don’t have to pass any parameters to enable it. This means the parameters <code>num_beams</code> is set to 1 and <code>do_sample=False</code>.",qt,de,Lt,me,At,he,ba=`The contrastive search decoding strategy was proposed in the 2022 paper <a href="https://arxiv.org/abs/2202.06417" rel="nofollow">A Contrastive Framework for Neural Text Generation</a>.
It demonstrates superior results for generating non-repetitive yet coherent long outputs. To learn how contrastive search
works, check out <a href="https://huggingface.co/blog/introducing-csearch" rel="nofollow">this blog post</a>.
The two main parameters that enable and control the behavior of contrastive search are <code>penalty_alpha</code> and <code>top_k</code>:`,Dt,ge,Kt,ue,Pt,ye,fa=`As opposed to greedy search that always chooses a token with the highest probability as the
next token, multinomial sampling (also called ancestral sampling) randomly selects the next token based on the probability distribution over the entire
vocabulary given by the model. Every token with a non-zero probability has a chance of being selected, thus reducing the
risk of repetition.`,Ot,Me,wa="To enable multinomial sampling set <code>do_sample=True</code> and <code>num_beams=1</code>.",es,be,ts,fe,ss,we,Ta=`Unlike greedy search, beam-search decoding keeps several hypotheses at each time step and eventually chooses
the hypothesis that has the overall highest probability for the entire sequence. This has the advantage of identifying high-probability
sequences that start with lower probability initial tokens and would’ve been ignored by the greedy search.`,as,T,ja='<img style="max-width: 90%; margin: auto;" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/beam_search.png"/>',ns,Te,Ja='You can visualize how beam-search decoding works in <a href="https://huggingface.co/spaces/m-ric/beam_search_visualizer" rel="nofollow">this interactive demo</a>: type your input sentence, and play with the parameters to see how the decoding beams change.',ls,je,Ua="To enable this decoding strategy, specify the <code>num_beams</code> (aka number of hypotheses to keep track of) that is greater than 1.",os,Je,is,Ue,rs,ke,ka=`As the name implies, this decoding strategy combines beam search with multinomial sampling. You need to specify
the <code>num_beams</code> greater than 1, and set <code>do_sample=True</code> to use this decoding strategy.`,ps,Ze,cs,ve,ds,We,Za=`The diverse beam search decoding strategy is an extension of the beam search strategy that allows for generating a more diverse
set of beam sequences to choose from. To learn how it works, refer to <a href="https://arxiv.org/pdf/1610.02424.pdf" rel="nofollow">Diverse Beam Search: Decoding Diverse Solutions from Neural Sequence Models</a>.
This approach has three main parameters: <code>num_beams</code>, <code>num_beam_groups</code>, and <code>diversity_penalty</code>.
The diversity penalty ensures the outputs are distinct across groups, and beam search is used within each group.`,ms,Ge,hs,Be,va=`This guide illustrates the main parameters that enable various decoding strategies. More advanced parameters exist for the
<code>generate</code> method, which gives you even further control over the <code>generate</code> method’s behavior.
For the complete list of the available parameters, refer to the <a href="./main_classes/text_generation">API documentation</a>.`,gs,Xe,us,Ce,Wa=`Speculative decoding (also known as assisted decoding) is a modification of the decoding strategies above, that uses an
assistant model (ideally a much smaller one), to generate a few candidate tokens. The main model then validates the candidate
tokens in a single forward pass, which speeds up the decoding process. If <code>do_sample=True</code>, then the token validation with
resampling introduced in the <a href="https://arxiv.org/pdf/2211.17192.pdf" rel="nofollow">speculative decoding paper</a> is used.
Assisted decoding assumes the main and assistant models have the same tokenizer, otherwise, see Universal Assisted Decoding below.`,ys,Ve,Ga=`Currently, only greedy search and sampling are supported with assisted decoding, and assisted decoding doesn’t support batched inputs.
To learn more about assisted decoding, check <a href="https://huggingface.co/blog/assisted-generation" rel="nofollow">this blog post</a>.`,Ms,Re,Ba="To enable assisted decoding, set the <code>assistant_model</code> argument with a model.",bs,_e,fs,Ie,Xa=`When using assisted decoding with sampling methods, you can use the <code>temperature</code> argument to control the randomness,
just like in multinomial sampling. However, in assisted decoding, reducing the temperature may help improve the latency.`,ws,$e,Ts,He,js,Ne,Ca=`Universal Assisted Decoding (UAD) adds support for main and assistant models with different tokenizers.
To use it, simply pass the tokenizers using the <code>tokenizer</code> and <code>assistant_tokenizer</code> arguments (see below).
Internally, the main model input tokens are re-encoded into assistant model tokens, then candidate tokens are generated in the assistant encoding, which are
in turn re-encoded into main model candidate tokens. Validation then proceeds as explained above.
The re-encoding steps involve decoding token ids into text and then encoding the text using a different tokenizer.
Since re-encoding the tokens may result in tokenization discrepancies, UAD finds the longest common subsequence between the source and target encodings,
to ensure the new tokens include the correct prompt suffix.`,Js,ze,Us,xe,ks,Fe,Va=`Alternatively, you can also set the <code>prompt_lookup_num_tokens</code> to trigger n-gram based assisted decoding, as opposed
to model based assisted decoding. You can read more about it <a href="https://twitter.com/joao_gante/status/1747322413006643259" rel="nofollow">here</a>.`,Zs,Qe,vs,Ye,Ra=`An LLM can be trained to also use its language modeling head with earlier hidden states as input, effectively
skipping layers to yield a lower-quality output — a technique called early exiting.
We use the lower-quality early exit output as an assistant output, and apply self-speculation to fix the output using the remaining layers. The final generation of that self-speculative solution is the same (or has the same distribution) as the original model’s generation.
If the model you’re using was trained to do early exit, you can pass
<code>assistant_early_exit</code> (integer). In this case, the assistant model will be the same model but exiting early, hence the
“self-speculative” name. Because the assistant model is a portion of the target model, caches and weights can be shared, which results in lower memory requirements. As in other assisted generation methods, the final generated result has the same quality as if no assistant had been used.`,Ws,Se,Gs,Ee,Bs,qe,_a=`<strong>D</strong>ecoding by C<strong>o</strong>ntrasting <strong>La</strong>yers (DoLa) is a contrastive decoding strategy to improve the factuality and reduce the
hallucinations of LLMs, as described in this paper of ICLR 2024 <a href="https://arxiv.org/abs/2309.03883" rel="nofollow">DoLa: Decoding by Contrasting Layers Improves Factuality in Large Language Models</a>.`,Xs,Le,Ia=`DoLa is achieved by contrasting the differences in logits obtained from final
layers versus earlier layers, thus amplify the factual knowledge localized to particular part of transformer layers.`,Cs,Ae,$a="Do the following two steps to activate DoLa decoding when calling the <code>model.generate</code> function:",Vs,De,Ha="<li>Set the <code>dola_layers</code> argument, which can be either a string or a list of integers.<ul><li>If set to a string, it can be one of <code>low</code>, <code>high</code>.</li> <li>If set to a list of integers, it should be a list of layer indices between 0 and the total number of layers in the model. The 0-th layer is word embedding, and the 1st layer is the first transformer layer, and so on.</li></ul></li> <li>Set <code>repetition_penalty = 1.2</code> is suggested to reduce repetition in DoLa decoding.</li>",Rs,Ke,Na="See the following examples for DoLa decoding with the 32-layer LLaMA-7B model.",_s,Pe,Is,Oe,$s,et,za="<code>dola_layers</code> stands for the candidate layers in premature layer selection, as described in the DoLa paper. The selected premature layer will be contrasted with the final layer.",Hs,tt,xa="Setting <code>dola_layers</code> to <code>&#39;low&#39;</code> or <code>&#39;high&#39;</code> will select the lower or higher part of the layers to contrast, respectively.",Ns,st,Fa="<li>For <code>N</code>-layer models with <code>N &lt;= 40</code> layers, the layers of <code>range(0, N // 2, 2)</code> and <code>range(N // 2, N, 2)</code> are used for <code>&#39;low&#39;</code> and <code>&#39;high&#39;</code> layers, respectively.</li> <li>For models with <code>N &gt; 40</code> layers, the layers of <code>range(0, 20, 2)</code> and <code>range(N - 20, N, 2)</code> are used for <code>&#39;low&#39;</code> and <code>&#39;high&#39;</code> layers, respectively.</li> <li>If the model has tied word embeddings, we skip the word embeddings (0-th) layer and start from the 2nd layer, as the early exit from word embeddings will become identity function.</li> <li>Set the <code>dola_layers</code> to a list of integers for layer indices to contrast manually specified layers. For example, setting <code>dola_layers=[28,30]</code> will contrast the final layer (32-th layer) with the 28-th and 30-th layers.</li>",zs,at,Qa="The paper suggested that contrasting <code>&#39;high&#39;</code> layers to improve short-answer tasks like TruthfulQA, and contrasting <code>&#39;low&#39;</code> layers to improve all the other long-answer reasoning tasks, such as GSM8K, StrategyQA, FACTOR, and VicunaQA. Applying DoLa to smaller models like GPT-2 is not recommended, as the results shown in the Appendix N of the paper.",xs,nt,Fs,lt,Qs;return Z=new M({props:{title:"Text generation strategies",local:"text-generation-strategies",headingTag:"h1"}}),R=new M({props:{title:"Default text generation configuration",local:"default-text-generation-configuration",headingTag:"h2"}}),$=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyZGlzdGlsYmVydCUyRmRpc3RpbGdwdDIlMjIpJTBBbW9kZWwuZ2VuZXJhdGlvbl9jb25maWc=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;distilbert/distilgpt2&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.generation_config
GenerationConfig {
  <span class="hljs-string">&quot;bos_token_id&quot;</span>: <span class="hljs-number">50256</span>,
  <span class="hljs-string">&quot;eos_token_id&quot;</span>: <span class="hljs-number">50256</span>
}
&lt;BLANKLINE&gt;`,wrap:!1}}),z=new M({props:{title:"Customize text generation",local:"customize-text-generation",headingTag:"h2"}}),F=new b({props:{code:"bXlfbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBudW1fYmVhbXMlM0Q0JTJDJTIwZG9fc2FtcGxlJTNEVHJ1ZSk=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>my_model.generate(**inputs, num_beams=<span class="hljs-number">4</span>, do_sample=<span class="hljs-literal">True</span>)',wrap:!1}}),S=new M({props:{title:"Save a custom decoding strategy with your model",local:"save-a-custom-decoding-strategy-with-your-model",headingTag:"h2"}}),L=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwR2VuZXJhdGlvbkNvbmZpZyUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMm15X2FjY291bnQlMkZteV9tb2RlbCUyMiklMEFnZW5lcmF0aW9uX2NvbmZpZyUyMCUzRCUyMEdlbmVyYXRpb25Db25maWcoJTBBJTIwJTIwJTIwJTIwbWF4X25ld190b2tlbnMlM0Q1MCUyQyUyMGRvX3NhbXBsZSUzRFRydWUlMkMlMjB0b3BfayUzRDUwJTJDJTIwZW9zX3Rva2VuX2lkJTNEbW9kZWwuY29uZmlnLmVvc190b2tlbl9pZCUwQSklMEFnZW5lcmF0aW9uX2NvbmZpZy5zYXZlX3ByZXRyYWluZWQoJTIybXlfYWNjb3VudCUyRm15X21vZGVsJTIyJTJDJTIwcHVzaF90b19odWIlM0RUcnVlKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, GenerationConfig

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;my_account/my_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>generation_config = GenerationConfig(
<span class="hljs-meta">... </span>    max_new_tokens=<span class="hljs-number">50</span>, do_sample=<span class="hljs-literal">True</span>, top_k=<span class="hljs-number">50</span>, eos_token_id=model.config.eos_token_id
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>generation_config.save_pretrained(<span class="hljs-string">&quot;my_account/my_model&quot;</span>, push_to_hub=<span class="hljs-literal">True</span>)`,wrap:!1}}),D=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclNlcTJTZXFMTSUyQyUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBHZW5lcmF0aW9uQ29uZmlnJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlLXQ1JTJGdDUtc21hbGwlMjIpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JTZXEyU2VxTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZS10NSUyRnQ1LXNtYWxsJTIyKSUwQSUwQXRyYW5zbGF0aW9uX2dlbmVyYXRpb25fY29uZmlnJTIwJTNEJTIwR2VuZXJhdGlvbkNvbmZpZyglMEElMjAlMjAlMjAlMjBudW1fYmVhbXMlM0Q0JTJDJTBBJTIwJTIwJTIwJTIwZWFybHlfc3RvcHBpbmclM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwZGVjb2Rlcl9zdGFydF90b2tlbl9pZCUzRDAlMkMlMEElMjAlMjAlMjAlMjBlb3NfdG9rZW5faWQlM0Rtb2RlbC5jb25maWcuZW9zX3Rva2VuX2lkJTJDJTBBJTIwJTIwJTIwJTIwcGFkX3Rva2VuJTNEbW9kZWwuY29uZmlnLnBhZF90b2tlbl9pZCUyQyUwQSklMEElMEElMjMlMjBUaXAlM0ElMjBhZGQlMjAlNjBwdXNoX3RvX2h1YiUzRFRydWUlNjAlMjB0byUyMHB1c2glMjB0byUyMHRoZSUyMEh1YiUwQXRyYW5zbGF0aW9uX2dlbmVyYXRpb25fY29uZmlnLnNhdmVfcHJldHJhaW5lZCglMjIlMkZ0bXAlMjIlMkMlMjAlMjJ0cmFuc2xhdGlvbl9nZW5lcmF0aW9uX2NvbmZpZy5qc29uJTIyKSUwQSUwQSUyMyUyMFlvdSUyMGNvdWxkJTIwdGhlbiUyMHVzZSUyMHRoZSUyMG5hbWVkJTIwZ2VuZXJhdGlvbiUyMGNvbmZpZyUyMGZpbGUlMjB0byUyMHBhcmFtZXRlcml6ZSUyMGdlbmVyYXRpb24lMEFnZW5lcmF0aW9uX2NvbmZpZyUyMCUzRCUyMEdlbmVyYXRpb25Db25maWcuZnJvbV9wcmV0cmFpbmVkKCUyMiUyRnRtcCUyMiUyQyUyMCUyMnRyYW5zbGF0aW9uX2dlbmVyYXRpb25fY29uZmlnLmpzb24lMjIpJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMnRyYW5zbGF0ZSUyMEVuZ2xpc2glMjB0byUyMEZyZW5jaCUzQSUyMENvbmZpZ3VyYXRpb24lMjBmaWxlcyUyMGFyZSUyMGVhc3klMjB0byUyMHVzZSElMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMGdlbmVyYXRpb25fY29uZmlnJTNEZ2VuZXJhdGlvbl9jb25maWcpJTBBcHJpbnQodG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXRwdXRzJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSeq2SeqLM, AutoTokenizer, GenerationConfig

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;google-t5/t5-small&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSeq2SeqLM.from_pretrained(<span class="hljs-string">&quot;google-t5/t5-small&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>translation_generation_config = GenerationConfig(
<span class="hljs-meta">... </span>    num_beams=<span class="hljs-number">4</span>,
<span class="hljs-meta">... </span>    early_stopping=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    decoder_start_token_id=<span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>    eos_token_id=model.config.eos_token_id,
<span class="hljs-meta">... </span>    pad_token=model.config.pad_token_id,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Tip: add \`push_to_hub=True\` to push to the Hub</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>translation_generation_config.save_pretrained(<span class="hljs-string">&quot;/tmp&quot;</span>, <span class="hljs-string">&quot;translation_generation_config.json&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># You could then use the named generation config file to parameterize generation</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generation_config = GenerationConfig.from_pretrained(<span class="hljs-string">&quot;/tmp&quot;</span>, <span class="hljs-string">&quot;translation_generation_config.json&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(<span class="hljs-string">&quot;translate English to French: Configuration files are easy to use!&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(**inputs, generation_config=generation_config)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>))
[<span class="hljs-string">&#x27;Les fichiers de configuration sont faciles à utiliser!&#x27;</span>]`,wrap:!1}}),K=new M({props:{title:"Streaming",local:"streaming",headingTag:"h2"}}),J=new Ss({props:{warning:!0,$$slots:{default:[tn]},$$scope:{ctx:j}}}),ee=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMFRleHRTdHJlYW1lciUwQSUwQXRvayUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMm9wZW5haS1jb21tdW5pdHklMkZncHQyJTIyKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMm9wZW5haS1jb21tdW5pdHklMkZncHQyJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRvayglNUIlMjJBbiUyMGluY3JlYXNpbmclMjBzZXF1ZW5jZSUzQSUyMG9uZSUyQyUyMiU1RCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBc3RyZWFtZXIlMjAlM0QlMjBUZXh0U3RyZWFtZXIodG9rKSUwQSUwQSUyMyUyMERlc3BpdGUlMjByZXR1cm5pbmclMjB0aGUlMjB1c3VhbCUyMG91dHB1dCUyQyUyMHRoZSUyMHN0cmVhbWVyJTIwd2lsbCUyMGFsc28lMjBwcmludCUyMHRoZSUyMGdlbmVyYXRlZCUyMHRleHQlMjB0byUyMHN0ZG91dC4lMEFfJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBzdHJlYW1lciUzRHN0cmVhbWVyJTJDJTIwbWF4X25ld190b2tlbnMlM0QyMCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, TextStreamer

<span class="hljs-meta">&gt;&gt;&gt; </span>tok = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;openai-community/gpt2&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;openai-community/gpt2&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tok([<span class="hljs-string">&quot;An increasing sequence: one,&quot;</span>], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>streamer = TextStreamer(tok)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Despite returning the usual output, the streamer will also print the generated text to stdout.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>_ = model.generate(**inputs, streamer=streamer, max_new_tokens=<span class="hljs-number">20</span>)
An increasing sequence: one, two, three, four, five, six, seven, eight, nine, ten, eleven,`,wrap:!1}}),te=new M({props:{title:"Watermarking",local:"watermarking",headingTag:"h2"}}),U=new Ss({props:{warning:!0,$$slots:{default:[sn]},$$scope:{ctx:j}}}),le=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMFdhdGVybWFya0RldGVjdG9yJTJDJTIwV2F0ZXJtYXJraW5nQ29uZmlnJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyb3BlbmFpLWNvbW11bml0eSUyRmdwdDIlMjIpJTBBdG9rJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyb3BlbmFpLWNvbW11bml0eSUyRmdwdDIlMjIpJTBBdG9rLnBhZF90b2tlbl9pZCUyMCUzRCUyMHRvay5lb3NfdG9rZW5faWQlMEF0b2sucGFkZGluZ19zaWRlJTIwJTNEJTIwJTIybGVmdCUyMiUwQSUwQWlucHV0cyUyMCUzRCUyMHRvayglNUIlMjJUaGlzJTIwaXMlMjB0aGUlMjBiZWdpbm5pbmclMjBvZiUyMGElMjBsb25nJTIwc3RvcnklMjIlMkMlMjAlMjJBbGljZSUyMGFuZCUyMEJvYiUyMGFyZSUyMiU1RCUyQyUyMHBhZGRpbmclM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFpbnB1dF9sZW4lMjAlM0QlMjBpbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQuc2hhcGUlNUItMSU1RCUwQSUwQXdhdGVybWFya2luZ19jb25maWclMjAlM0QlMjBXYXRlcm1hcmtpbmdDb25maWcoYmlhcyUzRDIuNSUyQyUyMHNlZWRpbmdfc2NoZW1lJTNEJTIyc2VsZmhhc2glMjIpJTBBb3V0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjB3YXRlcm1hcmtpbmdfY29uZmlnJTNEd2F0ZXJtYXJraW5nX2NvbmZpZyUyQyUyMGRvX3NhbXBsZSUzREZhbHNlJTJDJTIwbWF4X2xlbmd0aCUzRDIwKSUwQSUwQWRldGVjdG9yJTIwJTNEJTIwV2F0ZXJtYXJrRGV0ZWN0b3IobW9kZWxfY29uZmlnJTNEbW9kZWwuY29uZmlnJTJDJTIwZGV2aWNlJTNEJTIyY3B1JTIyJTJDJTIwd2F0ZXJtYXJraW5nX2NvbmZpZyUzRHdhdGVybWFya2luZ19jb25maWcpJTBBZGV0ZWN0aW9uX291dCUyMCUzRCUyMGRldGVjdG9yKG91dCUyQyUyMHJldHVybl9kaWN0JTNEVHJ1ZSklMEFkZXRlY3Rpb25fb3V0LnByZWRpY3Rpb24=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM, WatermarkDetector, WatermarkingConfig

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;openai-community/gpt2&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tok = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;openai-community/gpt2&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tok.pad_token_id = tok.eos_token_id
<span class="hljs-meta">&gt;&gt;&gt; </span>tok.padding_side = <span class="hljs-string">&quot;left&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tok([<span class="hljs-string">&quot;This is the beginning of a long story&quot;</span>, <span class="hljs-string">&quot;Alice and Bob are&quot;</span>], padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>input_len = inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].shape[-<span class="hljs-number">1</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span>watermarking_config = WatermarkingConfig(bias=<span class="hljs-number">2.5</span>, seeding_scheme=<span class="hljs-string">&quot;selfhash&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>out = model.generate(**inputs, watermarking_config=watermarking_config, do_sample=<span class="hljs-literal">False</span>, max_length=<span class="hljs-number">20</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>detector = WatermarkDetector(model_config=model.config, device=<span class="hljs-string">&quot;cpu&quot;</span>, watermarking_config=watermarking_config)
<span class="hljs-meta">&gt;&gt;&gt; </span>detection_out = detector(out, return_dict=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>detection_out.prediction
array([<span class="hljs-literal">True</span>, <span class="hljs-literal">True</span>])`,wrap:!1}}),oe=new M({props:{title:"Decoding strategies",local:"decoding-strategies",headingTag:"h2"}}),k=new Ss({props:{$$slots:{default:[an]},$$scope:{ctx:j}}}),pe=new M({props:{title:"Greedy Search",local:"greedy-search",headingTag:"h3"}}),de=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQXByb21wdCUyMCUzRCUyMCUyMkklMjBsb29rJTIwZm9yd2FyZCUyMHRvJTIyJTBBY2hlY2twb2ludCUyMCUzRCUyMCUyMmRpc3RpbGJlcnQlMkZkaXN0aWxncHQyJTIyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIocHJvbXB0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyklMEF0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dHB1dHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;I look forward to&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>checkpoint = <span class="hljs-string">&quot;distilbert/distilgpt2&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(**inputs)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&#x27;I look forward to seeing you all again!\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n&#x27;</span>]`,wrap:!1}}),me=new M({props:{title:"Contrastive search",local:"contrastive-search",headingTag:"h3"}}),ge=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQSUwQWNoZWNrcG9pbnQlMjAlM0QlMjAlMjJvcGVuYWktY29tbXVuaXR5JTJGZ3B0Mi1sYXJnZSUyMiUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEElMEFwcm9tcHQlMjAlM0QlMjAlMjJIdWdnaW5nJTIwRmFjZSUyMENvbXBhbnklMjBpcyUyMiUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihwcm9tcHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMHBlbmFsdHlfYWxwaGElM0QwLjYlMkMlMjB0b3BfayUzRDQlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEwMCklMEF0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dHB1dHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>checkpoint = <span class="hljs-string">&quot;openai-community/gpt2-large&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(checkpoint)

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;Hugging Face Company is&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(**inputs, penalty_alpha=<span class="hljs-number">0.6</span>, top_k=<span class="hljs-number">4</span>, max_new_tokens=<span class="hljs-number">100</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&#x27;Hugging Face Company is a family owned and operated business. We pride ourselves on being the best
in the business and our customer service is second to none.\\n\\nIf you have any questions about our
products or services, feel free to contact us at any time. We look forward to hearing from you!&#x27;</span>]`,wrap:!1}}),ue=new M({props:{title:"Multinomial sampling",local:"multinomial-sampling",headingTag:"h3"}}),be=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMHNldF9zZWVkJTBBc2V0X3NlZWQoMCklMjAlMjAlMjMlMjBGb3IlMjByZXByb2R1Y2liaWxpdHklMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyb3BlbmFpLWNvbW11bml0eSUyRmdwdDItbGFyZ2UlMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBJTBBcHJvbXB0JTIwJTNEJTIwJTIyVG9kYXklMjB3YXMlMjBhbiUyMGFtYXppbmclMjBkYXklMjBiZWNhdXNlJTIyJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHByb21wdCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwZG9fc2FtcGxlJTNEVHJ1ZSUyQyUyMG51bV9iZWFtcyUzRDElMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEwMCklMEF0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dHB1dHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM, set_seed
<span class="hljs-meta">&gt;&gt;&gt; </span>set_seed(<span class="hljs-number">0</span>)  <span class="hljs-comment"># For reproducibility</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>checkpoint = <span class="hljs-string">&quot;openai-community/gpt2-large&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(checkpoint)

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;Today was an amazing day because&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(**inputs, do_sample=<span class="hljs-literal">True</span>, num_beams=<span class="hljs-number">1</span>, max_new_tokens=<span class="hljs-number">100</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&quot;Today was an amazing day because we received these wonderful items by the way of a gift shop. The box arrived on a Thursday and I opened it on Monday afternoon to receive the gifts. Both bags featured pieces from all the previous years!\\n\\nThe box had lots of surprises in it, including some sweet little mini chocolate chips! I don&#x27;t think I&#x27;d eat all of these. This was definitely one of the most expensive presents I have ever got, I actually got most of them for free!\\n\\nThe first package came&quot;</span>]`,wrap:!1}}),fe=new M({props:{title:"Beam-search decoding",local:"beam-search-decoding",headingTag:"h3"}}),Je=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQXByb21wdCUyMCUzRCUyMCUyMkl0JTIwaXMlMjBhc3RvbmlzaGluZyUyMGhvdyUyMG9uZSUyMGNhbiUyMiUwQWNoZWNrcG9pbnQlMjAlM0QlMjAlMjJvcGVuYWktY29tbXVuaXR5JTJGZ3B0Mi1tZWRpdW0lMjIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihwcm9tcHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwbnVtX2JlYW1zJTNENSUyQyUyMG1heF9uZXdfdG9rZW5zJTNENTApJTBBdG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXRwdXRzJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;It is astonishing how one can&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>checkpoint = <span class="hljs-string">&quot;openai-community/gpt2-medium&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(checkpoint)

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(**inputs, num_beams=<span class="hljs-number">5</span>, max_new_tokens=<span class="hljs-number">50</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&#x27;It is astonishing how one can have such a profound impact on the lives of so many people in such a short period of
time.&quot;\\n\\nHe added: &quot;I am very proud of the work I have been able to do in the last few years.\\n\\n&quot;I have&#x27;</span>]`,wrap:!1}}),Ue=new M({props:{title:"Beam-search multinomial sampling",local:"beam-search-multinomial-sampling",headingTag:"h3"}}),Ze=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JTZXEyU2VxTE0lMkMlMjBzZXRfc2VlZCUwQXNldF9zZWVkKDApJTIwJTIwJTIzJTIwRm9yJTIwcmVwcm9kdWNpYmlsaXR5JTBBJTBBcHJvbXB0JTIwJTNEJTIwJTIydHJhbnNsYXRlJTIwRW5nbGlzaCUyMHRvJTIwR2VybWFuJTNBJTIwVGhlJTIwaG91c2UlMjBpcyUyMHdvbmRlcmZ1bC4lMjIlMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZ29vZ2xlLXQ1JTJGdDUtc21hbGwlMjIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihwcm9tcHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxMlNlcUxNLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMG51bV9iZWFtcyUzRDUlMkMlMjBkb19zYW1wbGUlM0RUcnVlKSUwQXRva2VuaXplci5kZWNvZGUob3V0cHV0cyU1QjAlNUQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSeq2SeqLM, set_seed
<span class="hljs-meta">&gt;&gt;&gt; </span>set_seed(<span class="hljs-number">0</span>)  <span class="hljs-comment"># For reproducibility</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;translate English to German: The house is wonderful.&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>checkpoint = <span class="hljs-string">&quot;google-t5/t5-small&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint)

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(**inputs, num_beams=<span class="hljs-number">5</span>, do_sample=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(outputs[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>)
<span class="hljs-string">&#x27;Das Haus ist wunderbar.&#x27;</span>`,wrap:!1}}),ve=new M({props:{title:"Diverse beam search decoding",local:"diverse-beam-search-decoding",headingTag:"h3"}}),Ge=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JTZXEyU2VxTE0lMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZ29vZ2xlJTJGcGVnYXN1cy14c3VtJTIyJTBBcHJvbXB0JTIwJTNEJTIwKCUwQSUyMCUyMCUyMCUyMCUyMlRoZSUyMFBlcm1hY3VsdHVyZSUyMERlc2lnbiUyMFByaW5jaXBsZXMlMjBhcmUlMjBhJTIwc2V0JTIwb2YlMjB1bml2ZXJzYWwlMjBkZXNpZ24lMjBwcmluY2lwbGVzJTIwJTIyJTBBJTIwJTIwJTIwJTIwJTIydGhhdCUyMGNhbiUyMGJlJTIwYXBwbGllZCUyMHRvJTIwYW55JTIwbG9jYXRpb24lMkMlMjBjbGltYXRlJTIwYW5kJTIwY3VsdHVyZSUyQyUyMGFuZCUyMHRoZXklMjBhbGxvdyUyMHVzJTIwdG8lMjBkZXNpZ24lMjAlMjIlMEElMjAlMjAlMjAlMjAlMjJ0aGUlMjBtb3N0JTIwZWZmaWNpZW50JTIwYW5kJTIwc3VzdGFpbmFibGUlMjBodW1hbiUyMGhhYml0YXRpb24lMjBhbmQlMjBmb29kJTIwcHJvZHVjdGlvbiUyMHN5c3RlbXMuJTIwJTIyJTBBJTIwJTIwJTIwJTIwJTIyUGVybWFjdWx0dXJlJTIwaXMlMjBhJTIwZGVzaWduJTIwc3lzdGVtJTIwdGhhdCUyMGVuY29tcGFzc2VzJTIwYSUyMHdpZGUlMjB2YXJpZXR5JTIwb2YlMjBkaXNjaXBsaW5lcyUyQyUyMHN1Y2glMjAlMjIlMEElMjAlMjAlMjAlMjAlMjJhcyUyMGVjb2xvZ3klMkMlMjBsYW5kc2NhcGUlMjBkZXNpZ24lMkMlMjBlbnZpcm9ubWVudGFsJTIwc2NpZW5jZSUyMGFuZCUyMGVuZXJneSUyMGNvbnNlcnZhdGlvbiUyQyUyMGFuZCUyMHRoZSUyMCUyMiUwQSUyMCUyMCUyMCUyMCUyMlBlcm1hY3VsdHVyZSUyMGRlc2lnbiUyMHByaW5jaXBsZXMlMjBhcmUlMjBkcmF3biUyMGZyb20lMjB0aGVzZSUyMHZhcmlvdXMlMjBkaXNjaXBsaW5lcy4lMjBFYWNoJTIwaW5kaXZpZHVhbCUyMCUyMiUwQSUyMCUyMCUyMCUyMCUyMmRlc2lnbiUyMHByaW5jaXBsZSUyMGl0c2VsZiUyMGVtYm9kaWVzJTIwYSUyMGNvbXBsZXRlJTIwY29uY2VwdHVhbCUyMGZyYW1ld29yayUyMGJhc2VkJTIwb24lMjBzb3VuZCUyMCUyMiUwQSUyMCUyMCUyMCUyMCUyMnNjaWVudGlmaWMlMjBwcmluY2lwbGVzLiUyMFdoZW4lMjB3ZSUyMGJyaW5nJTIwYWxsJTIwdGhlc2UlMjBzZXBhcmF0ZSUyMCUyMHByaW5jaXBsZXMlMjB0b2dldGhlciUyQyUyMHdlJTIwY2FuJTIwJTIyJTBBJTIwJTIwJTIwJTIwJTIyY3JlYXRlJTIwYSUyMGRlc2lnbiUyMHN5c3RlbSUyMHRoYXQlMjBib3RoJTIwbG9va3MlMjBhdCUyMHdob2xlJTIwc3lzdGVtcyUyQyUyMHRoZSUyMHBhcnRzJTIwdGhhdCUyMHRoZXNlJTIwc3lzdGVtcyUyMCUyMiUwQSUyMCUyMCUyMCUyMCUyMmNvbnNpc3QlMjBvZiUyQyUyMGFuZCUyMGhvdyUyMHRob3NlJTIwcGFydHMlMjBpbnRlcmFjdCUyMHdpdGglMjBlYWNoJTIwb3RoZXIlMjB0byUyMGNyZWF0ZSUyMGElMjBjb21wbGV4JTJDJTIwZHluYW1pYyUyQyUyMCUyMiUwQSUyMCUyMCUyMCUyMCUyMmxpdmluZyUyMHN5c3RlbS4lMjBFYWNoJTIwZGVzaWduJTIwcHJpbmNpcGxlJTIwc2VydmVzJTIwYXMlMjBhJTIwdG9vbCUyMHRoYXQlMjBhbGxvd3MlMjB1cyUyMHRvJTIwaW50ZWdyYXRlJTIwYWxsJTIwJTIyJTBBJTIwJTIwJTIwJTIwJTIydGhlJTIwc2VwYXJhdGUlMjBwYXJ0cyUyMG9mJTIwYSUyMGRlc2lnbiUyQyUyMHJlZmVycmVkJTIwdG8lMjBhcyUyMGVsZW1lbnRzJTJDJTIwaW50byUyMGElMjBmdW5jdGlvbmFsJTJDJTIwc3luZXJnaXN0aWMlMkMlMjAlMjIlMEElMjAlMjAlMjAlMjAlMjJ3aG9sZSUyMHN5c3RlbSUyQyUyMHdoZXJlJTIwdGhlJTIwZWxlbWVudHMlMjBoYXJtb25pb3VzbHklMjBpbnRlcmFjdCUyMGFuZCUyMHdvcmslMjB0b2dldGhlciUyMGluJTIwdGhlJTIwbW9zdCUyMCUyMiUwQSUyMCUyMCUyMCUyMCUyMmVmZmljaWVudCUyMHdheSUyMHBvc3NpYmxlLiUyMiUwQSklMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihwcm9tcHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxMlNlcUxNLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMG51bV9iZWFtcyUzRDUlMkMlMjBudW1fYmVhbV9ncm91cHMlM0Q1JTJDJTIwbWF4X25ld190b2tlbnMlM0QzMCUyQyUyMGRpdmVyc2l0eV9wZW5hbHR5JTNEMS4wKSUwQXRva2VuaXplci5kZWNvZGUob3V0cHV0cyU1QjAlNUQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSeq2SeqLM

<span class="hljs-meta">&gt;&gt;&gt; </span>checkpoint = <span class="hljs-string">&quot;google/pegasus-xsum&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = (
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;The Permaculture Design Principles are a set of universal design principles &quot;</span>
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;that can be applied to any location, climate and culture, and they allow us to design &quot;</span>
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;the most efficient and sustainable human habitation and food production systems. &quot;</span>
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Permaculture is a design system that encompasses a wide variety of disciplines, such &quot;</span>
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;as ecology, landscape design, environmental science and energy conservation, and the &quot;</span>
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Permaculture design principles are drawn from these various disciplines. Each individual &quot;</span>
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;design principle itself embodies a complete conceptual framework based on sound &quot;</span>
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;scientific principles. When we bring all these separate  principles together, we can &quot;</span>
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;create a design system that both looks at whole systems, the parts that these systems &quot;</span>
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;consist of, and how those parts interact with each other to create a complex, dynamic, &quot;</span>
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;living system. Each design principle serves as a tool that allows us to integrate all &quot;</span>
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;the separate parts of a design, referred to as elements, into a functional, synergistic, &quot;</span>
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;whole system, where the elements harmoniously interact and work together in the most &quot;</span>
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;efficient way possible.&quot;</span>
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint)

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(**inputs, num_beams=<span class="hljs-number">5</span>, num_beam_groups=<span class="hljs-number">5</span>, max_new_tokens=<span class="hljs-number">30</span>, diversity_penalty=<span class="hljs-number">1.0</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(outputs[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>)
<span class="hljs-string">&#x27;The Design Principles are a set of universal design principles that can be applied to any location, climate and
culture, and they allow us to design the&#x27;</span>`,wrap:!1}}),Xe=new M({props:{title:"Speculative Decoding",local:"speculative-decoding",headingTag:"h3"}}),_e=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQXByb21wdCUyMCUzRCUyMCUyMkFsaWNlJTIwYW5kJTIwQm9iJTIyJTBBY2hlY2twb2ludCUyMCUzRCUyMCUyMkVsZXV0aGVyQUklMkZweXRoaWEtMS40Yi1kZWR1cGVkJTIyJTBBYXNzaXN0YW50X2NoZWNrcG9pbnQlMjAlM0QlMjAlMjJFbGV1dGhlckFJJTJGcHl0aGlhLTE2MG0tZGVkdXBlZCUyMiUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHByb21wdCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEFhc3Npc3RhbnRfbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoYXNzaXN0YW50X2NoZWNrcG9pbnQpJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwYXNzaXN0YW50X21vZGVsJTNEYXNzaXN0YW50X21vZGVsKSUwQXRva2VuaXplci5iYXRjaF9kZWNvZGUob3V0cHV0cyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;Alice and Bob&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>checkpoint = <span class="hljs-string">&quot;EleutherAI/pythia-1.4b-deduped&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>assistant_checkpoint = <span class="hljs-string">&quot;EleutherAI/pythia-160m-deduped&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>assistant_model = AutoModelForCausalLM.from_pretrained(assistant_checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(**inputs, assistant_model=assistant_model)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&#x27;Alice and Bob are sitting in a bar. Alice is drinking a beer and Bob is drinking a&#x27;</span>]`,wrap:!1}}),$e=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMHNldF9zZWVkJTBBc2V0X3NlZWQoNDIpJTIwJTIwJTIzJTIwRm9yJTIwcmVwcm9kdWNpYmlsaXR5JTBBJTBBcHJvbXB0JTIwJTNEJTIwJTIyQWxpY2UlMjBhbmQlMjBCb2IlMjIlMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyRWxldXRoZXJBSSUyRnB5dGhpYS0xLjRiLWRlZHVwZWQlMjIlMEFhc3Npc3RhbnRfY2hlY2twb2ludCUyMCUzRCUyMCUyMkVsZXV0aGVyQUklMkZweXRoaWEtMTYwbS1kZWR1cGVkJTIyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIocHJvbXB0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQWFzc2lzdGFudF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZChhc3Npc3RhbnRfY2hlY2twb2ludCklMEFvdXRwdXRzJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBhc3Npc3RhbnRfbW9kZWwlM0Rhc3Npc3RhbnRfbW9kZWwlMkMlMjBkb19zYW1wbGUlM0RUcnVlJTJDJTIwdGVtcGVyYXR1cmUlM0QwLjUpJTBBdG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXRwdXRzJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, set_seed
<span class="hljs-meta">&gt;&gt;&gt; </span>set_seed(<span class="hljs-number">42</span>)  <span class="hljs-comment"># For reproducibility</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;Alice and Bob&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>checkpoint = <span class="hljs-string">&quot;EleutherAI/pythia-1.4b-deduped&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>assistant_checkpoint = <span class="hljs-string">&quot;EleutherAI/pythia-160m-deduped&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>assistant_model = AutoModelForCausalLM.from_pretrained(assistant_checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(**inputs, assistant_model=assistant_model, do_sample=<span class="hljs-literal">True</span>, temperature=<span class="hljs-number">0.5</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&#x27;Alice and Bob, a couple of friends of mine, who are both in the same office as&#x27;</span>]`,wrap:!1}}),He=new M({props:{title:"Universal Assisted Decoding",local:"universal-assisted-decoding",headingTag:"h4"}}),ze=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQXByb21wdCUyMCUzRCUyMCUyMkFsaWNlJTIwYW5kJTIwQm9iJTIyJTBBY2hlY2twb2ludCUyMCUzRCUyMCUyMmdvb2dsZSUyRmdlbW1hLTItOWIlMjIlMEFhc3Npc3RhbnRfY2hlY2twb2ludCUyMCUzRCUyMCUyMmRvdWJsZTclMkZ2aWN1bmEtNjhtJTIyJTBBJTBBYXNzaXN0YW50X3Rva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKGFzc2lzdGFudF9jaGVja3BvaW50KSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHByb21wdCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEFhc3Npc3RhbnRfbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoYXNzaXN0YW50X2NoZWNrcG9pbnQpJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwYXNzaXN0YW50X21vZGVsJTNEYXNzaXN0YW50X21vZGVsJTJDJTIwdG9rZW5pemVyJTNEdG9rZW5pemVyJTJDJTIwYXNzaXN0YW50X3Rva2VuaXplciUzRGFzc2lzdGFudF90b2tlbml6ZXIpJTBBdG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXRwdXRzJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;Alice and Bob&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>checkpoint = <span class="hljs-string">&quot;google/gemma-2-9b&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>assistant_checkpoint = <span class="hljs-string">&quot;double7/vicuna-68m&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>assistant_tokenizer = AutoTokenizer.from_pretrained(assistant_checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>assistant_model = AutoModelForCausalLM.from_pretrained(assistant_checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(**inputs, assistant_model=assistant_model, tokenizer=tokenizer, assistant_tokenizer=assistant_tokenizer)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&#x27;Alice and Bob are sitting in a bar. Alice is drinking a beer and Bob is drinking a&#x27;</span>]`,wrap:!1}}),xe=new M({props:{title:"Prompt Lookup",local:"prompt-lookup",headingTag:"h4"}}),Qe=new M({props:{title:"Self-Speculative Decoding",local:"self-speculative-decoding",headingTag:"h4"}}),Se=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQXByb21wdCUyMCUzRCUyMCUyMkFsaWNlJTIwYW5kJTIwQm9iJTIyJTBBY2hlY2twb2ludCUyMCUzRCUyMCUyMmZhY2Vib29rJTJGbGF5ZXJza2lwLWxsYW1hMy4yLTFCJTIyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIocHJvbXB0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMGFzc2lzdGFudF9lYXJseV9leGl0JTNENCUyQyUyMGRvX3NhbXBsZSUzREZhbHNlJTJDJTIwbWF4X25ld190b2tlbnMlM0QyMCklMEF0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dHB1dHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;Alice and Bob&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>checkpoint = <span class="hljs-string">&quot;facebook/layerskip-llama3.2-1B&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(**inputs, assistant_early_exit=<span class="hljs-number">4</span>, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">20</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&#x27;Alice and Bob are sitting in a bar. Alice is drinking a beer and Bob is drinking a&#x27;</span>]`,wrap:!1}}),Ee=new M({props:{title:"DoLa Decoding",local:"dola-decoding",headingTag:"h3"}}),Pe=new b({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMHNldF9zZWVkJTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwYWNjZWxlcmF0ZS50ZXN0X3V0aWxzLnRlc3RpbmclMjBpbXBvcnQlMjBnZXRfYmFja2VuZCUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmh1Z2d5bGxhbWElMkZsbGFtYS03YiUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJodWdneWxsYW1hJTJGbGxhbWEtN2IlMjIlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYpJTBBZGV2aWNlJTJDJTIwXyUyQyUyMF8lMjAlM0QlMjBnZXRfYmFja2VuZCgpJTIwJTIzJTIwYXV0b21hdGljYWxseSUyMGRldGVjdHMlMjB0aGUlMjB1bmRlcmx5aW5nJTIwZGV2aWNlJTIwdHlwZSUyMChDVURBJTJDJTIwQ1BVJTJDJTIwWFBVJTJDJTIwTVBTJTJDJTIwZXRjLiklMEFtb2RlbC50byhkZXZpY2UpJTBBc2V0X3NlZWQoNDIpJTBBJTBBdGV4dCUyMCUzRCUyMCUyMk9uJTIwd2hhdCUyMGRhdGUlMjB3YXMlMjB0aGUlMjBEZWNsYXJhdGlvbiUyMG9mJTIwSW5kZXBlbmRlbmNlJTIwb2ZmaWNpYWxseSUyMHNpZ25lZCUzRiUyMiUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcih0ZXh0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8oZGV2aWNlKSUwQSUwQXZhbmlsbGFfb3V0cHV0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBkb19zYW1wbGUlM0RGYWxzZSUyQyUyMG1heF9uZXdfdG9rZW5zJTNENTApJTBBdG9rZW5pemVyLmJhdGNoX2RlY29kZSh2YW5pbGxhX291dHB1dCU1QiUzQSUyQyUyMGlucHV0cy5pbnB1dF9pZHMuc2hhcGUlNUItMSU1RCUzQSU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSUwQSUwQWRvbGFfaGlnaF9vdXRwdXQlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMGRvX3NhbXBsZSUzREZhbHNlJTJDJTIwbWF4X25ld190b2tlbnMlM0Q1MCUyQyUyMGRvbGFfbGF5ZXJzJTNEJ2hpZ2gnKSUwQXRva2VuaXplci5iYXRjaF9kZWNvZGUoZG9sYV9oaWdoX291dHB1dCU1QiUzQSUyQyUyMGlucHV0cy5pbnB1dF9pZHMuc2hhcGUlNUItMSU1RCUzQSU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSUwQSUwQWRvbGFfY3VzdG9tX291dHB1dCUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwZG9fc2FtcGxlJTNERmFsc2UlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDUwJTJDJTIwZG9sYV9sYXllcnMlM0QlNUIyOCUyQzMwJTVEJTJDJTIwcmVwZXRpdGlvbl9wZW5hbHR5JTNEMS4yKSUwQXRva2VuaXplci5iYXRjaF9kZWNvZGUoZG9sYV9jdXN0b21fb3V0cHV0JTVCJTNBJTJDJTIwaW5wdXRzLmlucHV0X2lkcy5zaGFwZSU1Qi0xJTVEJTNBJTVEJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM, set_seed
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> accelerate.test_utils.testing <span class="hljs-keyword">import</span> get_backend

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;huggyllama/llama-7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;huggyllama/llama-7b&quot;</span>, torch_dtype=torch.float16)
<span class="hljs-meta">&gt;&gt;&gt; </span>device, _, _ = get_backend() <span class="hljs-comment"># automatically detects the underlying device type (CUDA, CPU, XPU, MPS, etc.)</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.to(device)
<span class="hljs-meta">&gt;&gt;&gt; </span>set_seed(<span class="hljs-number">42</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>text = <span class="hljs-string">&quot;On what date was the Declaration of Independence officially signed?&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)

<span class="hljs-comment"># Vanilla greddy decoding</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>vanilla_output = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">50</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(vanilla_output[:, inputs.input_ids.shape[-<span class="hljs-number">1</span>]:], skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&#x27;\\nThe Declaration of Independence was signed on July 4, 1776.\\nWhat was the date of the signing of the Declaration of Independence?\\nThe Declaration of Independence was signed on July 4,&#x27;</span>]

<span class="hljs-comment"># DoLa decoding with contrasting higher part of layers (layers 16,18,...,30)</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dola_high_output = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">50</span>, dola_layers=<span class="hljs-string">&#x27;high&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(dola_high_output[:, inputs.input_ids.shape[-<span class="hljs-number">1</span>]:], skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&#x27;\\nJuly 4, 1776, when the Continental Congress voted to separate from Great Britain. The 56 delegates to the Continental Congress signed the Declaration on August 2, 1776.&#x27;</span>]

<span class="hljs-comment"># DoLa decoding with contrasting specific layers (layers 28 and 30)</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dola_custom_output = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">50</span>, dola_layers=[<span class="hljs-number">28</span>,<span class="hljs-number">30</span>], repetition_penalty=<span class="hljs-number">1.2</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(dola_custom_output[:, inputs.input_ids.shape[-<span class="hljs-number">1</span>]:], skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&#x27;\\nIt was officially signed on 2 August 1776, when 56 members of the Second Continental Congress, representing the original 13 American colonies, voted unanimously for the resolution for independence. The 2&#x27;</span>]`,wrap:!1}}),Oe=new M({props:{title:"Understanding the dola_layers argument",local:"understanding-the-dolalayers-argument",headingTag:"h4"}}),nt=new en({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/generation_strategies.md"}}),{c(){u=o("meta"),f=n(),y=o("p"),w=n(),p(Z.$$.fragment),ot=n(),v=o("p"),v.textContent=qs,it=n(),W=o("p"),W.innerHTML=Ls,rt=n(),G=o("ul"),G.innerHTML=As,pt=n(),B=o("p"),B.textContent=Ds,ct=n(),X=o("p"),X.innerHTML=Ks,dt=n(),C=o("p"),C.textContent=Ps,mt=n(),V=o("ul"),V.innerHTML=Os,ht=n(),p(R.$$.fragment),gt=n(),_=o("p"),_.innerHTML=ea,ut=n(),I=o("p"),I.innerHTML=ta,yt=n(),p($.$$.fragment),Mt=n(),H=o("p"),H.innerHTML=sa,bt=n(),N=o("p"),N.textContent=aa,ft=n(),p(z.$$.fragment),wt=n(),x=o("p"),x.innerHTML=na,Tt=n(),p(F.$$.fragment),jt=n(),Q=o("p"),Q.textContent=la,Jt=n(),Y=o("ul"),Y.innerHTML=oa,Ut=n(),p(S.$$.fragment),kt=n(),E=o("p"),E.textContent=ia,Zt=n(),q=o("ul"),q.innerHTML=ra,vt=n(),p(L.$$.fragment),Wt=n(),A=o("p"),A.innerHTML=pa,Gt=n(),p(D.$$.fragment),Bt=n(),p(K.$$.fragment),Xt=n(),P=o("p"),P.innerHTML=ca,Ct=n(),p(J.$$.fragment),Vt=n(),O=o("p"),O.innerHTML=da,Rt=n(),p(ee.$$.fragment),_t=n(),p(te.$$.fragment),It=n(),se=o("p"),se.innerHTML=ma,$t=n(),ae=o("p"),ae.innerHTML=ha,Ht=n(),p(U.$$.fragment),Nt=n(),ne=o("p"),ne.innerHTML=ga,zt=n(),p(le.$$.fragment),xt=n(),p(oe.$$.fragment),Ft=n(),ie=o("p"),ie.innerHTML=ua,Qt=n(),re=o("p"),re.textContent=ya,Yt=n(),p(k.$$.fragment),St=n(),p(pe.$$.fragment),Et=n(),ce=o("p"),ce.innerHTML=Ma,qt=n(),p(de.$$.fragment),Lt=n(),p(me.$$.fragment),At=n(),he=o("p"),he.innerHTML=ba,Dt=n(),p(ge.$$.fragment),Kt=n(),p(ue.$$.fragment),Pt=n(),ye=o("p"),ye.textContent=fa,Ot=n(),Me=o("p"),Me.innerHTML=wa,es=n(),p(be.$$.fragment),ts=n(),p(fe.$$.fragment),ss=n(),we=o("p"),we.textContent=Ta,as=n(),T=o("a"),T.innerHTML=ja,ns=n(),Te=o("p"),Te.innerHTML=Ja,ls=n(),je=o("p"),je.innerHTML=Ua,os=n(),p(Je.$$.fragment),is=n(),p(Ue.$$.fragment),rs=n(),ke=o("p"),ke.innerHTML=ka,ps=n(),p(Ze.$$.fragment),cs=n(),p(ve.$$.fragment),ds=n(),We=o("p"),We.innerHTML=Za,ms=n(),p(Ge.$$.fragment),hs=n(),Be=o("p"),Be.innerHTML=va,gs=n(),p(Xe.$$.fragment),us=n(),Ce=o("p"),Ce.innerHTML=Wa,ys=n(),Ve=o("p"),Ve.innerHTML=Ga,Ms=n(),Re=o("p"),Re.innerHTML=Ba,bs=n(),p(_e.$$.fragment),fs=n(),Ie=o("p"),Ie.innerHTML=Xa,ws=n(),p($e.$$.fragment),Ts=n(),p(He.$$.fragment),js=n(),Ne=o("p"),Ne.innerHTML=Ca,Js=n(),p(ze.$$.fragment),Us=n(),p(xe.$$.fragment),ks=n(),Fe=o("p"),Fe.innerHTML=Va,Zs=n(),p(Qe.$$.fragment),vs=n(),Ye=o("p"),Ye.innerHTML=Ra,Ws=n(),p(Se.$$.fragment),Gs=n(),p(Ee.$$.fragment),Bs=n(),qe=o("p"),qe.innerHTML=_a,Xs=n(),Le=o("p"),Le.textContent=Ia,Cs=n(),Ae=o("p"),Ae.innerHTML=$a,Vs=n(),De=o("ol"),De.innerHTML=Ha,Rs=n(),Ke=o("p"),Ke.textContent=Na,_s=n(),p(Pe.$$.fragment),Is=n(),p(Oe.$$.fragment),$s=n(),et=o("p"),et.innerHTML=za,Hs=n(),tt=o("p"),tt.innerHTML=xa,Ns=n(),st=o("ul"),st.innerHTML=Fa,zs=n(),at=o("p"),at.innerHTML=Qa,xs=n(),p(nt.$$.fragment),Fs=n(),lt=o("p"),this.h()},l(e){const t=Pa("svelte-u9bgzb",document.head);u=i(t,"META",{name:!0,content:!0}),t.forEach(s),f=l(e),y=i(e,"P",{}),qa(y).forEach(s),w=l(e),c(Z.$$.fragment,e),ot=l(e),v=i(e,"P",{"data-svelte-h":!0}),r(v)!=="svelte-1pq6r4w"&&(v.textContent=qs),it=l(e),W=i(e,"P",{"data-svelte-h":!0}),r(W)!=="svelte-1dqk32t"&&(W.innerHTML=Ls),rt=l(e),G=i(e,"UL",{"data-svelte-h":!0}),r(G)!=="svelte-18jzu0"&&(G.innerHTML=As),pt=l(e),B=i(e,"P",{"data-svelte-h":!0}),r(B)!=="svelte-5iqkcx"&&(B.textContent=Ds),ct=l(e),X=i(e,"P",{"data-svelte-h":!0}),r(X)!=="svelte-agd87v"&&(X.innerHTML=Ks),dt=l(e),C=i(e,"P",{"data-svelte-h":!0}),r(C)!=="svelte-1gun7m8"&&(C.textContent=Ps),mt=l(e),V=i(e,"UL",{"data-svelte-h":!0}),r(V)!=="svelte-l1azua"&&(V.innerHTML=Os),ht=l(e),c(R.$$.fragment,e),gt=l(e),_=i(e,"P",{"data-svelte-h":!0}),r(_)!=="svelte-eehu19"&&(_.innerHTML=ea),ut=l(e),I=i(e,"P",{"data-svelte-h":!0}),r(I)!=="svelte-2o7gdz"&&(I.innerHTML=ta),yt=l(e),c($.$$.fragment,e),Mt=l(e),H=i(e,"P",{"data-svelte-h":!0}),r(H)!=="svelte-18srzjs"&&(H.innerHTML=sa),bt=l(e),N=i(e,"P",{"data-svelte-h":!0}),r(N)!=="svelte-32rftl"&&(N.textContent=aa),ft=l(e),c(z.$$.fragment,e),wt=l(e),x=i(e,"P",{"data-svelte-h":!0}),r(x)!=="svelte-qprijs"&&(x.innerHTML=na),Tt=l(e),c(F.$$.fragment,e),jt=l(e),Q=i(e,"P",{"data-svelte-h":!0}),r(Q)!=="svelte-t9z639"&&(Q.textContent=la),Jt=l(e),Y=i(e,"UL",{"data-svelte-h":!0}),r(Y)!=="svelte-17ejddt"&&(Y.innerHTML=oa),Ut=l(e),c(S.$$.fragment,e),kt=l(e),E=i(e,"P",{"data-svelte-h":!0}),r(E)!=="svelte-1l1ri2h"&&(E.textContent=ia),Zt=l(e),q=i(e,"UL",{"data-svelte-h":!0}),r(q)!=="svelte-hk9wim"&&(q.innerHTML=ra),vt=l(e),c(L.$$.fragment,e),Wt=l(e),A=i(e,"P",{"data-svelte-h":!0}),r(A)!=="svelte-1fsi3zv"&&(A.innerHTML=pa),Gt=l(e),c(D.$$.fragment,e),Bt=l(e),c(K.$$.fragment,e),Xt=l(e),P=i(e,"P",{"data-svelte-h":!0}),r(P)!=="svelte-12rh7l9"&&(P.innerHTML=ca),Ct=l(e),c(J.$$.fragment,e),Vt=l(e),O=i(e,"P",{"data-svelte-h":!0}),r(O)!=="svelte-1luhkpq"&&(O.innerHTML=da),Rt=l(e),c(ee.$$.fragment,e),_t=l(e),c(te.$$.fragment,e),It=l(e),se=i(e,"P",{"data-svelte-h":!0}),r(se)!=="svelte-10ws557"&&(se.innerHTML=ma),$t=l(e),ae=i(e,"P",{"data-svelte-h":!0}),r(ae)!=="svelte-e5fdlr"&&(ae.innerHTML=ha),Ht=l(e),c(U.$$.fragment,e),Nt=l(e),ne=i(e,"P",{"data-svelte-h":!0}),r(ne)!=="svelte-ba6d9u"&&(ne.innerHTML=ga),zt=l(e),c(le.$$.fragment,e),xt=l(e),c(oe.$$.fragment,e),Ft=l(e),ie=i(e,"P",{"data-svelte-h":!0}),r(ie)!=="svelte-dhwbam"&&(ie.innerHTML=ua),Qt=l(e),re=i(e,"P",{"data-svelte-h":!0}),r(re)!=="svelte-nugt5b"&&(re.textContent=ya),Yt=l(e),c(k.$$.fragment,e),St=l(e),c(pe.$$.fragment,e),Et=l(e),ce=i(e,"P",{"data-svelte-h":!0}),r(ce)!=="svelte-1m7rj88"&&(ce.innerHTML=Ma),qt=l(e),c(de.$$.fragment,e),Lt=l(e),c(me.$$.fragment,e),At=l(e),he=i(e,"P",{"data-svelte-h":!0}),r(he)!=="svelte-m0y9j6"&&(he.innerHTML=ba),Dt=l(e),c(ge.$$.fragment,e),Kt=l(e),c(ue.$$.fragment,e),Pt=l(e),ye=i(e,"P",{"data-svelte-h":!0}),r(ye)!=="svelte-vsvvis"&&(ye.textContent=fa),Ot=l(e),Me=i(e,"P",{"data-svelte-h":!0}),r(Me)!=="svelte-ldtxsn"&&(Me.innerHTML=wa),es=l(e),c(be.$$.fragment,e),ts=l(e),c(fe.$$.fragment,e),ss=l(e),we=i(e,"P",{"data-svelte-h":!0}),r(we)!=="svelte-149ek3p"&&(we.textContent=Ta),as=l(e),T=i(e,"A",{href:!0,class:!0,"data-svelte-h":!0}),r(T)!=="svelte-1ck2qcm"&&(T.innerHTML=ja),ns=l(e),Te=i(e,"P",{"data-svelte-h":!0}),r(Te)!=="svelte-1ymxnsd"&&(Te.innerHTML=Ja),ls=l(e),je=i(e,"P",{"data-svelte-h":!0}),r(je)!=="svelte-krswod"&&(je.innerHTML=Ua),os=l(e),c(Je.$$.fragment,e),is=l(e),c(Ue.$$.fragment,e),rs=l(e),ke=i(e,"P",{"data-svelte-h":!0}),r(ke)!=="svelte-zgjlvh"&&(ke.innerHTML=ka),ps=l(e),c(Ze.$$.fragment,e),cs=l(e),c(ve.$$.fragment,e),ds=l(e),We=i(e,"P",{"data-svelte-h":!0}),r(We)!=="svelte-zdhf65"&&(We.innerHTML=Za),ms=l(e),c(Ge.$$.fragment,e),hs=l(e),Be=i(e,"P",{"data-svelte-h":!0}),r(Be)!=="svelte-pzkqt4"&&(Be.innerHTML=va),gs=l(e),c(Xe.$$.fragment,e),us=l(e),Ce=i(e,"P",{"data-svelte-h":!0}),r(Ce)!=="svelte-1ddqaeo"&&(Ce.innerHTML=Wa),ys=l(e),Ve=i(e,"P",{"data-svelte-h":!0}),r(Ve)!=="svelte-xy91hv"&&(Ve.innerHTML=Ga),Ms=l(e),Re=i(e,"P",{"data-svelte-h":!0}),r(Re)!=="svelte-ebd3ly"&&(Re.innerHTML=Ba),bs=l(e),c(_e.$$.fragment,e),fs=l(e),Ie=i(e,"P",{"data-svelte-h":!0}),r(Ie)!=="svelte-nm0fxt"&&(Ie.innerHTML=Xa),ws=l(e),c($e.$$.fragment,e),Ts=l(e),c(He.$$.fragment,e),js=l(e),Ne=i(e,"P",{"data-svelte-h":!0}),r(Ne)!=="svelte-zp0z7a"&&(Ne.innerHTML=Ca),Js=l(e),c(ze.$$.fragment,e),Us=l(e),c(xe.$$.fragment,e),ks=l(e),Fe=i(e,"P",{"data-svelte-h":!0}),r(Fe)!=="svelte-n7i3t9"&&(Fe.innerHTML=Va),Zs=l(e),c(Qe.$$.fragment,e),vs=l(e),Ye=i(e,"P",{"data-svelte-h":!0}),r(Ye)!=="svelte-1cb4g6u"&&(Ye.innerHTML=Ra),Ws=l(e),c(Se.$$.fragment,e),Gs=l(e),c(Ee.$$.fragment,e),Bs=l(e),qe=i(e,"P",{"data-svelte-h":!0}),r(qe)!=="svelte-c1qhny"&&(qe.innerHTML=_a),Xs=l(e),Le=i(e,"P",{"data-svelte-h":!0}),r(Le)!=="svelte-1fok5rt"&&(Le.textContent=Ia),Cs=l(e),Ae=i(e,"P",{"data-svelte-h":!0}),r(Ae)!=="svelte-1xb5ka9"&&(Ae.innerHTML=$a),Vs=l(e),De=i(e,"OL",{"data-svelte-h":!0}),r(De)!=="svelte-1rdm431"&&(De.innerHTML=Ha),Rs=l(e),Ke=i(e,"P",{"data-svelte-h":!0}),r(Ke)!=="svelte-17tp6io"&&(Ke.textContent=Na),_s=l(e),c(Pe.$$.fragment,e),Is=l(e),c(Oe.$$.fragment,e),$s=l(e),et=i(e,"P",{"data-svelte-h":!0}),r(et)!=="svelte-qgkuwa"&&(et.innerHTML=za),Hs=l(e),tt=i(e,"P",{"data-svelte-h":!0}),r(tt)!=="svelte-13a3h6h"&&(tt.innerHTML=xa),Ns=l(e),st=i(e,"UL",{"data-svelte-h":!0}),r(st)!=="svelte-bhw0w6"&&(st.innerHTML=Fa),zs=l(e),at=i(e,"P",{"data-svelte-h":!0}),r(at)!=="svelte-1v5aa4g"&&(at.innerHTML=Qa),xs=l(e),c(nt.$$.fragment,e),Fs=l(e),lt=i(e,"P",{}),qa(lt).forEach(s),this.h()},h(){Ys(u,"name","hf:doc:metadata"),Ys(u,"content",ln),Ys(T,"href","https://huggingface.co/spaces/m-ric/beam_search_visualizer"),Ys(T,"class","flex flex-col justify-center")},m(e,t){Oa(document.head,u),a(e,f,t),a(e,y,t),a(e,w,t),d(Z,e,t),a(e,ot,t),a(e,v,t),a(e,it,t),a(e,W,t),a(e,rt,t),a(e,G,t),a(e,pt,t),a(e,B,t),a(e,ct,t),a(e,X,t),a(e,dt,t),a(e,C,t),a(e,mt,t),a(e,V,t),a(e,ht,t),d(R,e,t),a(e,gt,t),a(e,_,t),a(e,ut,t),a(e,I,t),a(e,yt,t),d($,e,t),a(e,Mt,t),a(e,H,t),a(e,bt,t),a(e,N,t),a(e,ft,t),d(z,e,t),a(e,wt,t),a(e,x,t),a(e,Tt,t),d(F,e,t),a(e,jt,t),a(e,Q,t),a(e,Jt,t),a(e,Y,t),a(e,Ut,t),d(S,e,t),a(e,kt,t),a(e,E,t),a(e,Zt,t),a(e,q,t),a(e,vt,t),d(L,e,t),a(e,Wt,t),a(e,A,t),a(e,Gt,t),d(D,e,t),a(e,Bt,t),d(K,e,t),a(e,Xt,t),a(e,P,t),a(e,Ct,t),d(J,e,t),a(e,Vt,t),a(e,O,t),a(e,Rt,t),d(ee,e,t),a(e,_t,t),d(te,e,t),a(e,It,t),a(e,se,t),a(e,$t,t),a(e,ae,t),a(e,Ht,t),d(U,e,t),a(e,Nt,t),a(e,ne,t),a(e,zt,t),d(le,e,t),a(e,xt,t),d(oe,e,t),a(e,Ft,t),a(e,ie,t),a(e,Qt,t),a(e,re,t),a(e,Yt,t),d(k,e,t),a(e,St,t),d(pe,e,t),a(e,Et,t),a(e,ce,t),a(e,qt,t),d(de,e,t),a(e,Lt,t),d(me,e,t),a(e,At,t),a(e,he,t),a(e,Dt,t),d(ge,e,t),a(e,Kt,t),d(ue,e,t),a(e,Pt,t),a(e,ye,t),a(e,Ot,t),a(e,Me,t),a(e,es,t),d(be,e,t),a(e,ts,t),d(fe,e,t),a(e,ss,t),a(e,we,t),a(e,as,t),a(e,T,t),a(e,ns,t),a(e,Te,t),a(e,ls,t),a(e,je,t),a(e,os,t),d(Je,e,t),a(e,is,t),d(Ue,e,t),a(e,rs,t),a(e,ke,t),a(e,ps,t),d(Ze,e,t),a(e,cs,t),d(ve,e,t),a(e,ds,t),a(e,We,t),a(e,ms,t),d(Ge,e,t),a(e,hs,t),a(e,Be,t),a(e,gs,t),d(Xe,e,t),a(e,us,t),a(e,Ce,t),a(e,ys,t),a(e,Ve,t),a(e,Ms,t),a(e,Re,t),a(e,bs,t),d(_e,e,t),a(e,fs,t),a(e,Ie,t),a(e,ws,t),d($e,e,t),a(e,Ts,t),d(He,e,t),a(e,js,t),a(e,Ne,t),a(e,Js,t),d(ze,e,t),a(e,Us,t),d(xe,e,t),a(e,ks,t),a(e,Fe,t),a(e,Zs,t),d(Qe,e,t),a(e,vs,t),a(e,Ye,t),a(e,Ws,t),d(Se,e,t),a(e,Gs,t),d(Ee,e,t),a(e,Bs,t),a(e,qe,t),a(e,Xs,t),a(e,Le,t),a(e,Cs,t),a(e,Ae,t),a(e,Vs,t),a(e,De,t),a(e,Rs,t),a(e,Ke,t),a(e,_s,t),d(Pe,e,t),a(e,Is,t),d(Oe,e,t),a(e,$s,t),a(e,et,t),a(e,Hs,t),a(e,tt,t),a(e,Ns,t),a(e,st,t),a(e,zs,t),a(e,at,t),a(e,xs,t),d(nt,e,t),a(e,Fs,t),a(e,lt,t),Qs=!0},p(e,[t]){const Ya={};t&2&&(Ya.$$scope={dirty:t,ctx:e}),J.$set(Ya);const Sa={};t&2&&(Sa.$$scope={dirty:t,ctx:e}),U.$set(Sa);const Ea={};t&2&&(Ea.$$scope={dirty:t,ctx:e}),k.$set(Ea)},i(e){Qs||(m(Z.$$.fragment,e),m(R.$$.fragment,e),m($.$$.fragment,e),m(z.$$.fragment,e),m(F.$$.fragment,e),m(S.$$.fragment,e),m(L.$$.fragment,e),m(D.$$.fragment,e),m(K.$$.fragment,e),m(J.$$.fragment,e),m(ee.$$.fragment,e),m(te.$$.fragment,e),m(U.$$.fragment,e),m(le.$$.fragment,e),m(oe.$$.fragment,e),m(k.$$.fragment,e),m(pe.$$.fragment,e),m(de.$$.fragment,e),m(me.$$.fragment,e),m(ge.$$.fragment,e),m(ue.$$.fragment,e),m(be.$$.fragment,e),m(fe.$$.fragment,e),m(Je.$$.fragment,e),m(Ue.$$.fragment,e),m(Ze.$$.fragment,e),m(ve.$$.fragment,e),m(Ge.$$.fragment,e),m(Xe.$$.fragment,e),m(_e.$$.fragment,e),m($e.$$.fragment,e),m(He.$$.fragment,e),m(ze.$$.fragment,e),m(xe.$$.fragment,e),m(Qe.$$.fragment,e),m(Se.$$.fragment,e),m(Ee.$$.fragment,e),m(Pe.$$.fragment,e),m(Oe.$$.fragment,e),m(nt.$$.fragment,e),Qs=!0)},o(e){h(Z.$$.fragment,e),h(R.$$.fragment,e),h($.$$.fragment,e),h(z.$$.fragment,e),h(F.$$.fragment,e),h(S.$$.fragment,e),h(L.$$.fragment,e),h(D.$$.fragment,e),h(K.$$.fragment,e),h(J.$$.fragment,e),h(ee.$$.fragment,e),h(te.$$.fragment,e),h(U.$$.fragment,e),h(le.$$.fragment,e),h(oe.$$.fragment,e),h(k.$$.fragment,e),h(pe.$$.fragment,e),h(de.$$.fragment,e),h(me.$$.fragment,e),h(ge.$$.fragment,e),h(ue.$$.fragment,e),h(be.$$.fragment,e),h(fe.$$.fragment,e),h(Je.$$.fragment,e),h(Ue.$$.fragment,e),h(Ze.$$.fragment,e),h(ve.$$.fragment,e),h(Ge.$$.fragment,e),h(Xe.$$.fragment,e),h(_e.$$.fragment,e),h($e.$$.fragment,e),h(He.$$.fragment,e),h(ze.$$.fragment,e),h(xe.$$.fragment,e),h(Qe.$$.fragment,e),h(Se.$$.fragment,e),h(Ee.$$.fragment,e),h(Pe.$$.fragment,e),h(Oe.$$.fragment,e),h(nt.$$.fragment,e),Qs=!1},d(e){e&&(s(f),s(y),s(w),s(ot),s(v),s(it),s(W),s(rt),s(G),s(pt),s(B),s(ct),s(X),s(dt),s(C),s(mt),s(V),s(ht),s(gt),s(_),s(ut),s(I),s(yt),s(Mt),s(H),s(bt),s(N),s(ft),s(wt),s(x),s(Tt),s(jt),s(Q),s(Jt),s(Y),s(Ut),s(kt),s(E),s(Zt),s(q),s(vt),s(Wt),s(A),s(Gt),s(Bt),s(Xt),s(P),s(Ct),s(Vt),s(O),s(Rt),s(_t),s(It),s(se),s($t),s(ae),s(Ht),s(Nt),s(ne),s(zt),s(xt),s(Ft),s(ie),s(Qt),s(re),s(Yt),s(St),s(Et),s(ce),s(qt),s(Lt),s(At),s(he),s(Dt),s(Kt),s(Pt),s(ye),s(Ot),s(Me),s(es),s(ts),s(ss),s(we),s(as),s(T),s(ns),s(Te),s(ls),s(je),s(os),s(is),s(rs),s(ke),s(ps),s(cs),s(ds),s(We),s(ms),s(hs),s(Be),s(gs),s(us),s(Ce),s(ys),s(Ve),s(Ms),s(Re),s(bs),s(fs),s(Ie),s(ws),s(Ts),s(js),s(Ne),s(Js),s(Us),s(ks),s(Fe),s(Zs),s(vs),s(Ye),s(Ws),s(Gs),s(Bs),s(qe),s(Xs),s(Le),s(Cs),s(Ae),s(Vs),s(De),s(Rs),s(Ke),s(_s),s(Is),s($s),s(et),s(Hs),s(tt),s(Ns),s(st),s(zs),s(at),s(xs),s(Fs),s(lt)),s(u),g(Z,e),g(R,e),g($,e),g(z,e),g(F,e),g(S,e),g(L,e),g(D,e),g(K,e),g(J,e),g(ee,e),g(te,e),g(U,e),g(le,e),g(oe,e),g(k,e),g(pe,e),g(de,e),g(me,e),g(ge,e),g(ue,e),g(be,e),g(fe,e),g(Je,e),g(Ue,e),g(Ze,e),g(ve,e),g(Ge,e),g(Xe,e),g(_e,e),g($e,e),g(He,e),g(ze,e),g(xe,e),g(Qe,e),g(Se,e),g(Ee,e),g(Pe,e),g(Oe,e),g(nt,e)}}}const ln='{"title":"Text generation strategies","local":"text-generation-strategies","sections":[{"title":"Default text generation configuration","local":"default-text-generation-configuration","sections":[],"depth":2},{"title":"Customize text generation","local":"customize-text-generation","sections":[],"depth":2},{"title":"Save a custom decoding strategy with your model","local":"save-a-custom-decoding-strategy-with-your-model","sections":[],"depth":2},{"title":"Streaming","local":"streaming","sections":[],"depth":2},{"title":"Watermarking","local":"watermarking","sections":[],"depth":2},{"title":"Decoding strategies","local":"decoding-strategies","sections":[{"title":"Greedy Search","local":"greedy-search","sections":[],"depth":3},{"title":"Contrastive search","local":"contrastive-search","sections":[],"depth":3},{"title":"Multinomial sampling","local":"multinomial-sampling","sections":[],"depth":3},{"title":"Beam-search decoding","local":"beam-search-decoding","sections":[],"depth":3},{"title":"Beam-search multinomial sampling","local":"beam-search-multinomial-sampling","sections":[],"depth":3},{"title":"Diverse beam search decoding","local":"diverse-beam-search-decoding","sections":[],"depth":3},{"title":"Speculative Decoding","local":"speculative-decoding","sections":[{"title":"Universal Assisted Decoding","local":"universal-assisted-decoding","sections":[],"depth":4},{"title":"Prompt Lookup","local":"prompt-lookup","sections":[],"depth":4},{"title":"Self-Speculative Decoding","local":"self-speculative-decoding","sections":[],"depth":4}],"depth":3},{"title":"DoLa Decoding","local":"dola-decoding","sections":[{"title":"Understanding the dola_layers argument","local":"understanding-the-dolalayers-argument","sections":[],"depth":4}],"depth":3}],"depth":2}],"depth":1}';function on(j){return Aa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class hn extends Da{constructor(u){super(),Ka(this,u,on,nn,La,{})}}export{hn as component};
