import{s as se,o as te}from"../chunks/scheduler.37c15a92.js";import{S as ne,i as ae,g as M,s as p,r as c,A as pe,h as U,f as s,c as o,j as Pl,u,x as y,k as Ll,y as oe,a as n,v as m,t as r,b as Kl,d as J,w as T,p as le}from"../chunks/index.2bf4358c.js";import{C as w}from"../chunks/CodeBlock.4f5fc1ad.js";import{C as ee}from"../chunks/CourseFloatingBanner.06521e8b.js";import{F as ie}from"../chunks/FrameworkSwitchCourse.8d4d4ab6.js";import{H as Rl,E as re}from"../chunks/EditOnGithub.8916dc7c.js";function Je(j){let a,i;return a=new ee({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section6_tf.ipynb"}]}}),{c(){c(a.$$.fragment)},l(t){u(a.$$.fragment,t)},m(t,b){m(a,t,b),i=!0},i(t){i||(J(a.$$.fragment,t),i=!0)},o(t){r(a.$$.fragment,t),i=!1},d(t){T(a,t)}}}function ce(j){let a,i;return a=new ee({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section6_pt.ipynb"}]}}),{c(){c(a.$$.fragment)},l(t){u(a.$$.fragment,t)},m(t,b){m(a,t,b),i=!0},i(t){i||(J(a.$$.fragment,t),i=!0)},o(t){r(a.$$.fragment,t),i=!1},d(t){T(a,t)}}}function ue(j){let a,i;return a=new w({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMFRGQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbiUwQSUwQWNoZWNrcG9pbnQlMjAlM0QlMjAlMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZC1maW5ldHVuZWQtc3N0LTItZW5nbGlzaCUyMiUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBbW9kZWwlMjAlM0QlMjBURkF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBc2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEF0b2tlbnMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjB0cnVuY2F0aW9uJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIpJTBBb3V0cHV0JTIwJTNEJTIwbW9kZWwoKip0b2tlbnMp",highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
output = model(**tokens)`,wrap:!1}}),{c(){c(a.$$.fragment)},l(t){u(a.$$.fragment,t)},m(t,b){m(a,t,b),i=!0},i(t){i||(J(a.$$.fragment,t),i=!0)},o(t){r(a.$$.fragment,t),i=!1},d(t){T(a,t)}}}function me(j){let a,i;return a=new w({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbi5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEFzZXF1ZW5jZXMlMjAlM0QlMjAlNUIlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMkMlMjAlMjJTbyUyMGhhdmUlMjBJISUyMiU1RCUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFvdXRwdXQlMjAlM0QlMjBtb2RlbCgqKnRva2Vucyk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
output = model(**tokens)`,wrap:!1}}),{c(){c(a.$$.fragment)},l(t){u(a.$$.fragment,t)},m(t,b){m(a,t,b),i=!0},i(t){i||(J(a.$$.fragment,t),i=!0)},o(t){r(a.$$.fragment,t),i=!1},d(t){T(a,t)}}}function Te(j){let a,i,t,b,k,tl,g,nl,d,f,ll,V,Wl="在过去的几个章节中，我们已经尝试尽可能手动完成大部分工作。我们探索了 tokenizer 的运行机制，并且了解了分词、转换为 inputs ID、填充、截断以及注意力掩码的处理方式。",al,Z,Bl="然而，正如我们在第二节中看到的那样，🤗 Transformers API 能够通过一个高级函数为我们处理所有这些工作，接下来我们就要深入研究这个函数。当你直接在句子上调用你的 <code>tokenizer</code> 时，就可以得到转换后的可以直接放入模型的数据了：",pl,$,ol,N,Sl="这里， <code>model_inputs</code> 变量包含模型运行所需的所有数据。在 DistilBERT 中，它包括 inputs ID 和注意力掩码（attention mask）。其他接受额外输入的模型也会有对应的 tokenizer 可以将输入转化为模型所需要的输入。",il,E,zl="正如我们将在下面的一些例子中看到的，这个函数非常强大。首先，它可以对单个句子进行处理：",rl,Q,Jl,R,Cl="它还一次处理多个多个，并且 API 的用法完全一致：",cl,W,ul,B,_l="它可以使用多种不同的方法对目标进行填充：",ml,S,Tl,z,Gl="它还可以对序列进行截断：",Ml,C,Ul,_,Yl="<code>tokenizer</code> 对象可以处理指定框架张量的转换，然后可以直接发送到模型。例如，在下面的代码示例中，我们告诉 tokenizer 返回不同框架的张量 —— <code>&quot;pt&quot;</code> 返回 PyTorch 张量， <code>&quot;tf&quot;</code> 返回 TensorFlow 张量，而 <code>&quot;np&quot;</code> 则返回 NumPy 数组：",bl,G,yl,Y,wl,F,Fl="如果我们看一下 tokenizer 返回的 inputs ID，我们会发现它们与之前的略有不同：",jl,X,dl,x,fl,v,Xl="句子的开始和结束分别增加了一个 inputs ID。我们来解码上述的两个 ID 序列，看看是怎么回事：",hl,q,Il,H,kl,A,xl="tokenizer 在开头添加了特殊单词 <code>[CLS]</code> ，在结尾添加了特殊单词 <code>[SEP]</code> 。这是因为模型在预训练时使用了这些字词，所以为了得到相同的推断结果，我们也需要添加它们。请注意，有些模型不添加特殊单词，或者添加不同的特殊单词；模型也可能只在开头或结尾添加这些特殊单词。无论如何，tokenizer 知道哪些是必需的，并会为你处理这些问题。",gl,D,Vl,O,vl="现在我们已经看到 <code>tokenizer</code> 对象在处理文本时的所有步骤，让我们最后再看一次它如何通过 tokenizer  API 处理多个序列（填充！），非常长的序列（截断！）以及多种类型的张量：",Zl,h,I,el,P,$l,sl,Nl;k=new ie({props:{fw:j[0]}}),g=new Rl({props:{title:"综合应用",local:"综合应用",headingTag:"h1"}});const ql=[ce,Je],L=[];function Hl(l,e){return l[0]==="pt"?0:1}d=Hl(j),f=L[d]=ql[d](j),$=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQSUwQXNlcXVlbmNlJTIwJTNEJTIwJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTBBJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),Q=new w({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2Up",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),W=new w({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

model_inputs = tokenizer(sequences)`,wrap:!1}}),S=new w({props:{code:"JTIzJTIwJUU1JUIwJTg2JUU1JThGJUE1JUU1JUFEJTkwJUU1JUJBJThGJUU1JTg4JTk3JUU1JUExJUFCJUU1JTg1JTg1JUU1JTg4JUIwJUU2JTlDJTgwJUU5JTk1JUJGJUU1JThGJUE1JUU1JUFEJTkwJUU3JTlBJTg0JUU5JTk1JUJGJUU1JUJBJUE2JTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHBhZGRpbmclM0QlMjJsb25nZXN0JTIyKSUwQSUwQSUyMyUyMCVFNSVCMCU4NiVFNSU4RiVBNSVFNSVBRCU5MCVFNSVCQSU4RiVFNSU4OCU5NyVFNSVBMSVBQiVFNSU4NSU4NSVFNSU4OCVCMCVFNiVBOCVBMSVFNSU5RSU4QiVFNyU5QSU4NCVFNiU5QyU4MCVFNSVBNCVBNyVFOSU5NSVCRiVFNSVCQSVBNiUwQSUyMyUyMCg1MTIlMjBmb3IlMjBCRVJUJTIwb3IlMjBEaXN0aWxCRVJUKSUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEJTIybWF4X2xlbmd0aCUyMiklMEElMEElMjMlMjAlRTUlQjAlODYlRTUlOEYlQTUlRTUlQUQlOTAlRTUlQkElOEYlRTUlODglOTclRTUlQTElQUIlRTUlODUlODUlRTUlODglQjAlRTYlOEMlODclRTUlQUUlOUElRTclOUElODQlRTYlOUMlODAlRTUlQTQlQTclRTklOTUlQkYlRTUlQkElQTYlMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRCUyMm1heF9sZW5ndGglMjIlMkMlMjBtYXhfbGVuZ3RoJTNEOCk=",highlighted:`<span class="hljs-comment"># 将句子序列填充到最长句子的长度</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;longest&quot;</span>)

<span class="hljs-comment"># 将句子序列填充到模型的最大长度</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>)

<span class="hljs-comment"># 将句子序列填充到指定的最大长度</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>, max_length=<span class="hljs-number">8</span>)`,wrap:!1}}),C=new w({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjAlRTUlQjAlODYlRTYlODglQUElRTYlOTYlQUQlRTYlQUYlOTQlRTYlQTglQTElRTUlOUUlOEIlRTYlOUMlODAlRTUlQTQlQTclRTklOTUlQkYlRTUlQkElQTYlRTklOTUlQkYlRTclOUElODQlRTUlOEYlQTUlRTUlQUQlOTAlRTUlQkElOEYlRTUlODglOTclMEElMjMlMjAoNTEyJTIwZm9yJTIwQkVSVCUyMG9yJTIwRGlzdGlsQkVSVCklMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUpJTBBJTBBJTIzJTIwJUU1JUIwJTg2JUU2JTg4JUFBJUU2JTk2JUFEJUU5JTk1JUJGJUU0JUJBJThFJUU2JThDJTg3JUU1JUFFJTlBJUU2JTlDJTgwJUU1JUE0JUE3JUU5JTk1JUJGJUU1JUJBJUE2JUU3JTlBJTg0JUU1JThGJUE1JUU1JUFEJTkwJUU1JUJBJThGJUU1JTg4JTk3JTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMG1heF9sZW5ndGglM0Q4JTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUp",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># 将截断比模型最大长度长的句子序列</span>
<span class="hljs-comment"># (512 for BERT or DistilBERT)</span>
model_inputs = tokenizer(sequences, truncation=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># 将截断长于指定最大长度的句子序列</span>
model_inputs = tokenizer(sequences, max_length=<span class="hljs-number">8</span>, truncation=<span class="hljs-literal">True</span>)`,wrap:!1}}),G=new w({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjAlRTglQkYlOTQlRTUlOUIlOUUlMjBQeVRvcmNoJTIwdGVuc29ycyUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBJTIzJTIwJUU4JUJGJTk0JUU1JTlCJTlFJTIwVGVuc29yRmxvdyUyMHRlbnNvcnMlMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnRmJTIyKSUwQSUwQSUyMyUyMCVFOCVCRiU5NCVFNSU5QiU5RSUyME51bVB5JTIwYXJyYXlzJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHBhZGRpbmclM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJucCUyMik=",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># 返回 PyTorch tensors</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># 返回 TensorFlow tensors</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)

<span class="hljs-comment"># 返回 NumPy arrays</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)`,wrap:!1}}),Y=new Rl({props:{title:"特殊的 tokens",local:"特殊的 tokens",headingTag:"h2"}}),X=new w({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2UpJTBBcHJpbnQobW9kZWxfaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplci50b2tlbml6ZShzZXF1ZW5jZSklMEFpZHMlMjAlM0QlMjB0b2tlbml6ZXIuY29udmVydF90b2tlbnNfdG9faWRzKHRva2VucyklMEFwcmludChpZHMp",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)
<span class="hljs-built_in">print</span>(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])

tokens = tokenizer.tokenize(sequence)
ids = tokenizer.convert_tokens_to_ids(tokens)
<span class="hljs-built_in">print</span>(ids)`,wrap:!1}}),x=new w({props:{code:"JTVCMTAxJTJDJTIwMTA0NSUyQyUyMDEwMDUlMkMlMjAyMzEwJTJDJTIwMjA0MiUyQyUyMDM0MDMlMkMlMjAyMDA1JTJDJTIwMTAzNyUyQyUyMDE3NjYyJTJDJTIwMTIxNzIlMkMlMjAyNjA3JTJDJTIwMjAyNiUyQyUyMDI4NzglMkMlMjAyMTY2JTJDJTIwMTAxMiUyQyUyMDEwMiU1RCUwQSU1QjEwNDUlMkMlMjAxMDA1JTJDJTIwMjMxMCUyQyUyMDIwNDIlMkMlMjAzNDAzJTJDJTIwMjAwNSUyQyUyMDEwMzclMkMlMjAxNzY2MiUyQyUyMDEyMTcyJTJDJTIwMjYwNyUyQyUyMDIwMjYlMkMlMjAyODc4JTJDJTIwMjE2NiUyQyUyMDEwMTIlNUQ=",highlighted:`[<span class="hljs-number">101</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>]
[<span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>]`,wrap:!1}}),q=new w({props:{code:"cHJpbnQodG9rZW5pemVyLmRlY29kZShtb2RlbF9pbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQpKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUoaWRzKSk=",highlighted:`<span class="hljs-built_in">print</span>(tokenizer.decode(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>]))
<span class="hljs-built_in">print</span>(tokenizer.decode(ids))`,wrap:!1}}),H=new w({props:{code:"JTIyJTVCQ0xTJTVEJTIwaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIwJTVCU0VQJTVEJTIyJTBBJTIyaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIy",highlighted:`<span class="hljs-string">&quot;[CLS] i&#x27;ve been waiting for a huggingface course my whole life. [SEP]&quot;</span>
<span class="hljs-string">&quot;i&#x27;ve been waiting for a huggingface course my whole life.&quot;</span>`,wrap:!1}}),D=new Rl({props:{title:"小结：从 tokenizer 到模型",local:"小结：从 tokenizer 到模型",headingTag:"h2"}});const Al=[me,ue],K=[];function Dl(l,e){return l[0]==="pt"?0:1}return h=Dl(j),I=K[h]=Al[h](j),P=new re({props:{source:"https://github.com/huggingface/course/blob/main/chapters/zh-CN/chapter2/6.mdx"}}),{c(){a=M("meta"),i=p(),t=M("p"),b=p(),c(k.$$.fragment),tl=p(),c(g.$$.fragment),nl=p(),f.c(),ll=p(),V=M("p"),V.textContent=Wl,al=p(),Z=M("p"),Z.innerHTML=Bl,pl=p(),c($.$$.fragment),ol=p(),N=M("p"),N.innerHTML=Sl,il=p(),E=M("p"),E.textContent=zl,rl=p(),c(Q.$$.fragment),Jl=p(),R=M("p"),R.textContent=Cl,cl=p(),c(W.$$.fragment),ul=p(),B=M("p"),B.textContent=_l,ml=p(),c(S.$$.fragment),Tl=p(),z=M("p"),z.textContent=Gl,Ml=p(),c(C.$$.fragment),Ul=p(),_=M("p"),_.innerHTML=Yl,bl=p(),c(G.$$.fragment),yl=p(),c(Y.$$.fragment),wl=p(),F=M("p"),F.textContent=Fl,jl=p(),c(X.$$.fragment),dl=p(),c(x.$$.fragment),fl=p(),v=M("p"),v.textContent=Xl,hl=p(),c(q.$$.fragment),Il=p(),c(H.$$.fragment),kl=p(),A=M("p"),A.innerHTML=xl,gl=p(),c(D.$$.fragment),Vl=p(),O=M("p"),O.innerHTML=vl,Zl=p(),I.c(),el=p(),c(P.$$.fragment),$l=p(),sl=M("p"),this.h()},l(l){const e=pe("svelte-u9bgzb",document.head);a=U(e,"META",{name:!0,content:!0}),e.forEach(s),i=o(l),t=U(l,"P",{}),Pl(t).forEach(s),b=o(l),u(k.$$.fragment,l),tl=o(l),u(g.$$.fragment,l),nl=o(l),f.l(l),ll=o(l),V=U(l,"P",{"data-svelte-h":!0}),y(V)!=="svelte-khgo0k"&&(V.textContent=Wl),al=o(l),Z=U(l,"P",{"data-svelte-h":!0}),y(Z)!=="svelte-10bo7tr"&&(Z.innerHTML=Bl),pl=o(l),u($.$$.fragment,l),ol=o(l),N=U(l,"P",{"data-svelte-h":!0}),y(N)!=="svelte-1yxjh7p"&&(N.innerHTML=Sl),il=o(l),E=U(l,"P",{"data-svelte-h":!0}),y(E)!=="svelte-1yhxykx"&&(E.textContent=zl),rl=o(l),u(Q.$$.fragment,l),Jl=o(l),R=U(l,"P",{"data-svelte-h":!0}),y(R)!=="svelte-171v091"&&(R.textContent=Cl),cl=o(l),u(W.$$.fragment,l),ul=o(l),B=U(l,"P",{"data-svelte-h":!0}),y(B)!=="svelte-bj8bs1"&&(B.textContent=_l),ml=o(l),u(S.$$.fragment,l),Tl=o(l),z=U(l,"P",{"data-svelte-h":!0}),y(z)!=="svelte-u3ow8k"&&(z.textContent=Gl),Ml=o(l),u(C.$$.fragment,l),Ul=o(l),_=U(l,"P",{"data-svelte-h":!0}),y(_)!=="svelte-1nlo48x"&&(_.innerHTML=Yl),bl=o(l),u(G.$$.fragment,l),yl=o(l),u(Y.$$.fragment,l),wl=o(l),F=U(l,"P",{"data-svelte-h":!0}),y(F)!=="svelte-isnfts"&&(F.textContent=Fl),jl=o(l),u(X.$$.fragment,l),dl=o(l),u(x.$$.fragment,l),fl=o(l),v=U(l,"P",{"data-svelte-h":!0}),y(v)!=="svelte-1z0h8tc"&&(v.textContent=Xl),hl=o(l),u(q.$$.fragment,l),Il=o(l),u(H.$$.fragment,l),kl=o(l),A=U(l,"P",{"data-svelte-h":!0}),y(A)!=="svelte-vskbu9"&&(A.innerHTML=xl),gl=o(l),u(D.$$.fragment,l),Vl=o(l),O=U(l,"P",{"data-svelte-h":!0}),y(O)!=="svelte-zcw8dw"&&(O.innerHTML=vl),Zl=o(l),I.l(l),el=o(l),u(P.$$.fragment,l),$l=o(l),sl=U(l,"P",{}),Pl(sl).forEach(s),this.h()},h(){Ll(a,"name","hf:doc:metadata"),Ll(a,"content",Me)},m(l,e){oe(document.head,a),n(l,i,e),n(l,t,e),n(l,b,e),m(k,l,e),n(l,tl,e),m(g,l,e),n(l,nl,e),L[d].m(l,e),n(l,ll,e),n(l,V,e),n(l,al,e),n(l,Z,e),n(l,pl,e),m($,l,e),n(l,ol,e),n(l,N,e),n(l,il,e),n(l,E,e),n(l,rl,e),m(Q,l,e),n(l,Jl,e),n(l,R,e),n(l,cl,e),m(W,l,e),n(l,ul,e),n(l,B,e),n(l,ml,e),m(S,l,e),n(l,Tl,e),n(l,z,e),n(l,Ml,e),m(C,l,e),n(l,Ul,e),n(l,_,e),n(l,bl,e),m(G,l,e),n(l,yl,e),m(Y,l,e),n(l,wl,e),n(l,F,e),n(l,jl,e),m(X,l,e),n(l,dl,e),m(x,l,e),n(l,fl,e),n(l,v,e),n(l,hl,e),m(q,l,e),n(l,Il,e),m(H,l,e),n(l,kl,e),n(l,A,e),n(l,gl,e),m(D,l,e),n(l,Vl,e),n(l,O,e),n(l,Zl,e),K[h].m(l,e),n(l,el,e),m(P,l,e),n(l,$l,e),n(l,sl,e),Nl=!0},p(l,[e]){const Ol={};e&1&&(Ol.fw=l[0]),k.$set(Ol);let El=d;d=Hl(l),d!==El&&(le(),r(L[El],1,1,()=>{L[El]=null}),Kl(),f=L[d],f||(f=L[d]=ql[d](l),f.c()),J(f,1),f.m(ll.parentNode,ll));let Ql=h;h=Dl(l),h!==Ql&&(le(),r(K[Ql],1,1,()=>{K[Ql]=null}),Kl(),I=K[h],I||(I=K[h]=Al[h](l),I.c()),J(I,1),I.m(el.parentNode,el))},i(l){Nl||(J(k.$$.fragment,l),J(g.$$.fragment,l),J(f),J($.$$.fragment,l),J(Q.$$.fragment,l),J(W.$$.fragment,l),J(S.$$.fragment,l),J(C.$$.fragment,l),J(G.$$.fragment,l),J(Y.$$.fragment,l),J(X.$$.fragment,l),J(x.$$.fragment,l),J(q.$$.fragment,l),J(H.$$.fragment,l),J(D.$$.fragment,l),J(I),J(P.$$.fragment,l),Nl=!0)},o(l){r(k.$$.fragment,l),r(g.$$.fragment,l),r(f),r($.$$.fragment,l),r(Q.$$.fragment,l),r(W.$$.fragment,l),r(S.$$.fragment,l),r(C.$$.fragment,l),r(G.$$.fragment,l),r(Y.$$.fragment,l),r(X.$$.fragment,l),r(x.$$.fragment,l),r(q.$$.fragment,l),r(H.$$.fragment,l),r(D.$$.fragment,l),r(I),r(P.$$.fragment,l),Nl=!1},d(l){l&&(s(i),s(t),s(b),s(tl),s(nl),s(ll),s(V),s(al),s(Z),s(pl),s(ol),s(N),s(il),s(E),s(rl),s(Jl),s(R),s(cl),s(ul),s(B),s(ml),s(Tl),s(z),s(Ml),s(Ul),s(_),s(bl),s(yl),s(wl),s(F),s(jl),s(dl),s(fl),s(v),s(hl),s(Il),s(kl),s(A),s(gl),s(Vl),s(O),s(Zl),s(el),s($l),s(sl)),s(a),T(k,l),T(g,l),L[d].d(l),T($,l),T(Q,l),T(W,l),T(S,l),T(C,l),T(G,l),T(Y,l),T(X,l),T(x,l),T(q,l),T(H,l),T(D,l),K[h].d(l),T(P,l)}}}const Me='{"title":"综合应用","local":"综合应用","sections":[{"title":"特殊的 tokens","local":"特殊的 tokens","sections":[],"depth":2},{"title":"小结：从 tokenizer 到模型","local":"小结：从 tokenizer 到模型","sections":[],"depth":2}],"depth":1}';function Ue(j,a,i){let t="pt";return te(()=>{const b=new URLSearchParams(window.location.search);i(0,t=b.get("fw")||"pt")}),[t]}class he extends ne{constructor(a){super(),ae(this,a,Ue,Te,se,{})}}export{he as component};
