import{s as F,n as G,o as K}from"../chunks/scheduler.36a0863c.js";import{S as N,i as O,g as f,s,r as w,A as R,h as k,f as a,c as m,j as A,u as M,x as y,k as B,y as W,a as n,v as T,d as E,t as H,w as L}from"../chunks/index.156fee99.js";import{C as D}from"../chunks/CourseFloatingBanner.16bb8bff.js";import{H as I,E as J}from"../chunks/getInferenceSnippets.472bc46d.js";function Q(S){let i,$,h,d,l,b,r,_,o,U="Kerja bagus telah menyelesaikan bab ini!",z,u,j="Setelah menyelami dunia tokenizer secara mendalam, kini kamu seharusnya:",v,p,q="<li>Mampu melatih tokenizer baru menggunakan tokenizer lama sebagai template</li> <li>Memahami cara menggunakan offset untuk memetakan posisi token ke bagian asli dalam teks</li> <li>Mengetahui perbedaan antara BPE, WordPiece, dan Unigram</li> <li>Mampu mencampur dan mencocokkan blok yang disediakan oleh pustaka 🤗 Tokenizers untuk membangun tokenizer versimu sendiri</li> <li>Mampu menggunakan tokenizer tersebut dalam pustaka 🤗 Transformers</li>",x,c,C,g,P;return l=new I({props:{title:"Tokenizer, selesai!",local:"tokenizers-check",headingTag:"h1"}}),r=new D({props:{chapter:6,classNames:"absolute z-10 right-0 top-0"}}),c=new J({props:{source:"https://github.com/huggingface/course/blob/main/chapters/id/chapter6/9.mdx"}}),{c(){i=f("meta"),$=s(),h=f("p"),d=s(),w(l.$$.fragment),b=s(),w(r.$$.fragment),_=s(),o=f("p"),o.textContent=U,z=s(),u=f("p"),u.textContent=j,v=s(),p=f("ul"),p.innerHTML=q,x=s(),w(c.$$.fragment),C=s(),g=f("p"),this.h()},l(e){const t=R("svelte-u9bgzb",document.head);i=k(t,"META",{name:!0,content:!0}),t.forEach(a),$=m(e),h=k(e,"P",{}),A(h).forEach(a),d=m(e),M(l.$$.fragment,e),b=m(e),M(r.$$.fragment,e),_=m(e),o=k(e,"P",{"data-svelte-h":!0}),y(o)!=="svelte-1vqtnnt"&&(o.textContent=U),z=m(e),u=k(e,"P",{"data-svelte-h":!0}),y(u)!=="svelte-1cwg5bz"&&(u.textContent=j),v=m(e),p=k(e,"UL",{"data-svelte-h":!0}),y(p)!=="svelte-1nr2t2g"&&(p.innerHTML=q),x=m(e),M(c.$$.fragment,e),C=m(e),g=k(e,"P",{}),A(g).forEach(a),this.h()},h(){B(i,"name","hf:doc:metadata"),B(i,"content",V)},m(e,t){W(document.head,i),n(e,$,t),n(e,h,t),n(e,d,t),T(l,e,t),n(e,b,t),T(r,e,t),n(e,_,t),n(e,o,t),n(e,z,t),n(e,u,t),n(e,v,t),n(e,p,t),n(e,x,t),T(c,e,t),n(e,C,t),n(e,g,t),P=!0},p:G,i(e){P||(E(l.$$.fragment,e),E(r.$$.fragment,e),E(c.$$.fragment,e),P=!0)},o(e){H(l.$$.fragment,e),H(r.$$.fragment,e),H(c.$$.fragment,e),P=!1},d(e){e&&(a($),a(h),a(d),a(b),a(_),a(o),a(z),a(u),a(v),a(p),a(x),a(C),a(g)),a(i),L(l,e),L(r,e),L(c,e)}}}const V='{"title":"Tokenizer, selesai!","local":"tokenizers-check","sections":[],"depth":1}';function X(S){return K(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ae extends N{constructor(i){super(),O(this,i,X,Q,F,{})}}export{ae as component};
