import{s as ce,n as he,o as de}from"../chunks/scheduler.25b97de1.js";import{S as ke,i as ge,g as r,s as a,r as m,A as ye,h as i,f as s,c as l,j as pe,u as p,x as f,k as fe,y as Me,a as n,v as c,d as h,t as d,w as k}from"../chunks/index.d9030fc9.js";import{C as D}from"../chunks/CodeBlock.e6cd0d95.js";import{H as ee,E as ue}from"../chunks/EditOnGithub.91d95064.js";function je(te){let o,C,B,I,g,P,y,se=`The <a href="/docs/transformers/pr_29693/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> depends on the <a href="https://huggingface.co/docs/tokenizers" rel="nofollow">🤗 Tokenizers</a> library. The tokenizers obtained from the 🤗 Tokenizers library can be
loaded very simply into 🤗 Transformers.`,W,M,ne="Before getting in the specifics, let’s first start by creating a dummy tokenizer in a few lines:",N,u,Q,j,ae=`We now have a tokenizer trained on the files we defined. We can either continue using it in that runtime, or save it to
a JSON file for future re-use.`,E,z,G,b,le=`Let’s see how to leverage this tokenizer object in the 🤗 Transformers library. The
<a href="/docs/transformers/pr_29693/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> class allows for easy instantiation, by accepting the instantiated
<em>tokenizer</em> object as an argument:`,X,T,H,$,re=`This object can now be used with all the methods shared by the 🤗 Transformers tokenizers! Head to <a href="main_classes/tokenizer">the tokenizer
page</a> for more information.`,L,w,R,U,ie="In order to load a tokenizer from a JSON file, let’s first start by saving our tokenizer:",q,Z,x,_,oe=`The path to which we saved this file can be passed to the <a href="/docs/transformers/pr_29693/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> initialization
method using the <code>tokenizer_file</code> parameter:`,S,J,A,V,me=`This object can now be used with all the methods shared by the 🤗 Transformers tokenizers! Head to <a href="main_classes/tokenizer">the tokenizer
page</a> for more information.`,Y,v,K,F,O;return g=new ee({props:{title:"Use tokenizers from 🤗 Tokenizers",local:"use-tokenizers-from--tokenizers",headingTag:"h1"}}),u=new D({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMEFmcm9tJTIwdG9rZW5pemVycy5tb2RlbHMlMjBpbXBvcnQlMjBCUEUlMEFmcm9tJTIwdG9rZW5pemVycy50cmFpbmVycyUyMGltcG9ydCUyMEJwZVRyYWluZXIlMEFmcm9tJTIwdG9rZW5pemVycy5wcmVfdG9rZW5pemVycyUyMGltcG9ydCUyMFdoaXRlc3BhY2UlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBUb2tlbml6ZXIoQlBFKHVua190b2tlbiUzRCUyMiU1QlVOSyU1RCUyMikpJTBBdHJhaW5lciUyMCUzRCUyMEJwZVRyYWluZXIoc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlNUJVTkslNUQlMjIlMkMlMjAlMjIlNUJDTFMlNUQlMjIlMkMlMjAlMjIlNUJTRVAlNUQlMjIlMkMlMjAlMjIlNUJQQUQlNUQlMjIlMkMlMjAlMjIlNUJNQVNLJTVEJTIyJTVEKSUwQSUwQXRva2VuaXplci5wcmVfdG9rZW5pemVyJTIwJTNEJTIwV2hpdGVzcGFjZSgpJTBBZmlsZXMlMjAlM0QlMjAlNUIuLi4lNUQlMEF0b2tlbml6ZXIudHJhaW4oZmlsZXMlMkMlMjB0cmFpbmVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> BPE
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> BpeTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.pre_tokenizers <span class="hljs-keyword">import</span> Whitespace

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(BPE(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = BpeTrainer(special_tokens=[<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pre_tokenizer = Whitespace()
<span class="hljs-meta">&gt;&gt;&gt; </span>files = [...]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train(files, trainer)`,wrap:!1}}),z=new ee({props:{title:"Loading directly from the tokenizer object",local:"loading-directly-from-the-tokenizer-object",headingTag:"h2"}}),T=new D({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfb2JqZWN0JTNEdG9rZW5pemVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_object=tokenizer)`,wrap:!1}}),w=new ee({props:{title:"Loading from a JSON file",local:"loading-from-a-json-file",headingTag:"h2"}}),Z=new D({props:{code:"dG9rZW5pemVyLnNhdmUoJTIydG9rZW5pemVyLmpzb24lMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.save(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)',wrap:!1}}),J=new D({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfZmlsZSUzRCUyMnRva2VuaXplci5qc29uJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_file=<span class="hljs-string">&quot;tokenizer.json&quot;</span>)`,wrap:!1}}),v=new ue({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/fast_tokenizers.md"}}),{c(){o=r("meta"),C=a(),B=r("p"),I=a(),m(g.$$.fragment),P=a(),y=r("p"),y.innerHTML=se,W=a(),M=r("p"),M.textContent=ne,N=a(),m(u.$$.fragment),Q=a(),j=r("p"),j.textContent=ae,E=a(),m(z.$$.fragment),G=a(),b=r("p"),b.innerHTML=le,X=a(),m(T.$$.fragment),H=a(),$=r("p"),$.innerHTML=re,L=a(),m(w.$$.fragment),R=a(),U=r("p"),U.textContent=ie,q=a(),m(Z.$$.fragment),x=a(),_=r("p"),_.innerHTML=oe,S=a(),m(J.$$.fragment),A=a(),V=r("p"),V.innerHTML=me,Y=a(),m(v.$$.fragment),K=a(),F=r("p"),this.h()},l(e){const t=ye("svelte-u9bgzb",document.head);o=i(t,"META",{name:!0,content:!0}),t.forEach(s),C=l(e),B=i(e,"P",{}),pe(B).forEach(s),I=l(e),p(g.$$.fragment,e),P=l(e),y=i(e,"P",{"data-svelte-h":!0}),f(y)!=="svelte-ykyrlh"&&(y.innerHTML=se),W=l(e),M=i(e,"P",{"data-svelte-h":!0}),f(M)!=="svelte-1fy1h9g"&&(M.textContent=ne),N=l(e),p(u.$$.fragment,e),Q=l(e),j=i(e,"P",{"data-svelte-h":!0}),f(j)!=="svelte-rt9ca6"&&(j.textContent=ae),E=l(e),p(z.$$.fragment,e),G=l(e),b=i(e,"P",{"data-svelte-h":!0}),f(b)!=="svelte-1ak8mcr"&&(b.innerHTML=le),X=l(e),p(T.$$.fragment,e),H=l(e),$=i(e,"P",{"data-svelte-h":!0}),f($)!=="svelte-mowuiv"&&($.innerHTML=re),L=l(e),p(w.$$.fragment,e),R=l(e),U=i(e,"P",{"data-svelte-h":!0}),f(U)!=="svelte-kuw3zs"&&(U.textContent=ie),q=l(e),p(Z.$$.fragment,e),x=l(e),_=i(e,"P",{"data-svelte-h":!0}),f(_)!=="svelte-6yr1fn"&&(_.innerHTML=oe),S=l(e),p(J.$$.fragment,e),A=l(e),V=i(e,"P",{"data-svelte-h":!0}),f(V)!=="svelte-mowuiv"&&(V.innerHTML=me),Y=l(e),p(v.$$.fragment,e),K=l(e),F=i(e,"P",{}),pe(F).forEach(s),this.h()},h(){fe(o,"name","hf:doc:metadata"),fe(o,"content",ze)},m(e,t){Me(document.head,o),n(e,C,t),n(e,B,t),n(e,I,t),c(g,e,t),n(e,P,t),n(e,y,t),n(e,W,t),n(e,M,t),n(e,N,t),c(u,e,t),n(e,Q,t),n(e,j,t),n(e,E,t),c(z,e,t),n(e,G,t),n(e,b,t),n(e,X,t),c(T,e,t),n(e,H,t),n(e,$,t),n(e,L,t),c(w,e,t),n(e,R,t),n(e,U,t),n(e,q,t),c(Z,e,t),n(e,x,t),n(e,_,t),n(e,S,t),c(J,e,t),n(e,A,t),n(e,V,t),n(e,Y,t),c(v,e,t),n(e,K,t),n(e,F,t),O=!0},p:he,i(e){O||(h(g.$$.fragment,e),h(u.$$.fragment,e),h(z.$$.fragment,e),h(T.$$.fragment,e),h(w.$$.fragment,e),h(Z.$$.fragment,e),h(J.$$.fragment,e),h(v.$$.fragment,e),O=!0)},o(e){d(g.$$.fragment,e),d(u.$$.fragment,e),d(z.$$.fragment,e),d(T.$$.fragment,e),d(w.$$.fragment,e),d(Z.$$.fragment,e),d(J.$$.fragment,e),d(v.$$.fragment,e),O=!1},d(e){e&&(s(C),s(B),s(I),s(P),s(y),s(W),s(M),s(N),s(Q),s(j),s(E),s(G),s(b),s(X),s(H),s($),s(L),s(R),s(U),s(q),s(x),s(_),s(S),s(A),s(V),s(Y),s(K),s(F)),s(o),k(g,e),k(u,e),k(z,e),k(T,e),k(w,e),k(Z,e),k(J,e),k(v,e)}}}const ze='{"title":"Use tokenizers from 🤗 Tokenizers","local":"use-tokenizers-from--tokenizers","sections":[{"title":"Loading directly from the tokenizer object","local":"loading-directly-from-the-tokenizer-object","sections":[],"depth":2},{"title":"Loading from a JSON file","local":"loading-from-a-json-file","sections":[],"depth":2}],"depth":1}';function be(te){return de(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ze extends ke{constructor(o){super(),ge(this,o,be,je,ce,{})}}export{Ze as component};
