import{s as N,n as F,o as G}from"../chunks/scheduler.37c15a92.js";import{S as O,i as R,g as c,s,r as w,A as W,h as $,f as n,c as l,j as A,u as E,x as y,k as B,y as D,a,v as T,d as H,t as L,w as M}from"../chunks/index.2bf4358c.js";import{C as I}from"../chunks/CourseFloatingBanner.06521e8b.js";import{H as J,E as K}from"../chunks/EditOnGithub.8916dc7c.js";function Q(S){let i,d,h,z,r,g,o,k,m,U="恭喜你完成了这一章！",x,p,j="在深入研究 tokenizer 之后，你应该：",C,f,q="<li>能够使用旧的 tokenizer 作为模板来训练新的 tokenizer</li> <li>了解如何使用偏移量将 tokens 的位置映射到其原始文本范围</li> <li>了解 BPE、WordPiece 和 Unigram 之间的区别</li> <li>能够混合使用 🤗 Tokenizers 库提供的块来构建你自己的 tokenizer</li> <li>能够在 🤗 Transformers 库中使用该 tokenizer</li>",v,u,P,_,b;return r=new J({props:{title:"tokenizer 回顾！",local:"tokenizer ，回顾！",headingTag:"h1"}}),o=new I({props:{chapter:6,classNames:"absolute z-10 right-0 top-0"}}),u=new K({props:{source:"https://github.com/huggingface/course/blob/main/chapters/zh-CN/chapter6/9.mdx"}}),{c(){i=c("meta"),d=s(),h=c("p"),z=s(),w(r.$$.fragment),g=s(),w(o.$$.fragment),k=s(),m=c("p"),m.textContent=U,x=s(),p=c("p"),p.textContent=j,C=s(),f=c("ul"),f.innerHTML=q,v=s(),w(u.$$.fragment),P=s(),_=c("p"),this.h()},l(t){const e=W("svelte-u9bgzb",document.head);i=$(e,"META",{name:!0,content:!0}),e.forEach(n),d=l(t),h=$(t,"P",{}),A(h).forEach(n),z=l(t),E(r.$$.fragment,t),g=l(t),E(o.$$.fragment,t),k=l(t),m=$(t,"P",{"data-svelte-h":!0}),y(m)!=="svelte-wo5o3f"&&(m.textContent=U),x=l(t),p=$(t,"P",{"data-svelte-h":!0}),y(p)!=="svelte-ykyhjl"&&(p.textContent=j),C=l(t),f=$(t,"UL",{"data-svelte-h":!0}),y(f)!=="svelte-7z1lnq"&&(f.innerHTML=q),v=l(t),E(u.$$.fragment,t),P=l(t),_=$(t,"P",{}),A(_).forEach(n),this.h()},h(){B(i,"name","hf:doc:metadata"),B(i,"content",V)},m(t,e){D(document.head,i),a(t,d,e),a(t,h,e),a(t,z,e),T(r,t,e),a(t,g,e),T(o,t,e),a(t,k,e),a(t,m,e),a(t,x,e),a(t,p,e),a(t,C,e),a(t,f,e),a(t,v,e),T(u,t,e),a(t,P,e),a(t,_,e),b=!0},p:F,i(t){b||(H(r.$$.fragment,t),H(o.$$.fragment,t),H(u.$$.fragment,t),b=!0)},o(t){L(r.$$.fragment,t),L(o.$$.fragment,t),L(u.$$.fragment,t),b=!1},d(t){t&&(n(d),n(h),n(z),n(g),n(k),n(m),n(x),n(p),n(C),n(f),n(v),n(P),n(_)),n(i),M(r,t),M(o,t),M(u,t)}}}const V='{"title":"tokenizer 回顾！","local":"tokenizer ，回顾！","sections":[],"depth":1}';function X(S){return G(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class nt extends O{constructor(i){super(),R(this,i,X,Q,N,{})}}export{nt as component};
