import{s as He,o as Ne}from"../chunks/scheduler.36a0863c.js";import{S as Qe,i as _e,g as M,s as t,r as i,A as Fe,h as b,f as n,c as s,j as Ce,u as p,x as c,k as qe,y as Ae,a as l,v as u,d as m,t as o,w as r}from"../chunks/index.156fee99.js";import{C as d}from"../chunks/CodeBlock.4cf998e6.js";import{C as De}from"../chunks/CourseFloatingBanner.16bb8bff.js";import{F as Pe}from"../chunks/FrameworkSwitchCourse.d8719b67.js";import{H as Ze,E as Le}from"../chunks/getInferenceSnippets.472bc46d.js";function Ke(Ue){let j,F,y,A,k,P,g,L,J,K,h,$e="Dalam beberapa bagian sebelumnya, kita telah mencoba melakukan sebagian besar proses secara manual. Kita telah mempelajari cara kerja tokenizer dan mengeksplorasi tokenisasi, konversi ke input ID, padding, pemotongan (truncation), dan attention mask.",O,T,We="Namun seperti yang telah kita lihat di bagian 2, API 🤗 Transformers bisa menangani semua ini secara otomatis melalui fungsi tingkat tinggi yang akan kita bahas di sini. Saat Anda memanggil <code>tokenizer</code> langsung pada kalimat, Anda akan mendapatkan input yang siap digunakan oleh model:",ee,w,ae,I,Ve="Di sini, variabel <code>model_inputs</code> sudah berisi semua hal yang diperlukan agar model dapat berfungsi dengan baik. Untuk DistilBERT, ini mencakup <code>input_ids</code> dan <code>attention_mask</code>. Model lain yang memerlukan input tambahan juga akan diberikan oleh objek <code>tokenizer</code>.",ne,f,Be="Seperti yang akan kita lihat dalam contoh-contoh berikut, metode ini sangat kuat. Pertama, ia bisa menangani satu urutan:",le,U,te,Z,ze="Ia juga bisa menangani beberapa urutan sekaligus, tanpa mengubah API:",se,$,ie,W,Ee="Ia juga bisa melakukan padding sesuai dengan berbagai tujuan:",pe,V,ue,B,Ye="Ia juga bisa memotong (truncate) urutan:",me,z,oe,E,Se="Objek <code>tokenizer</code> dapat menangani konversi ke tensor dari framework tertentu, yang kemudian dapat langsung dikirim ke model. Sebagai contoh, pada cuplikan kode berikut kita meminta tokenizer untuk mengembalikan tensor dari berbagai framework — <code>&quot;pt&quot;</code> akan mengembalikan tensor PyTorch dan <code>&quot;np&quot;</code> akan mengembalikan array NumPy:",re,Y,Me,S,be,X,Xe="Jika kita melihat <code>input_ids</code> yang dikembalikan oleh tokenizer, kita akan melihat perbedaan kecil dibandingkan sebelumnya:",ce,G,de,R,je,v,Ge="Satu token ID ditambahkan di awal dan satu di akhir. Mari kita dekode keduanya untuk melihat apa bedanya:",ye,x,ke,C,ge,q,Re="Tokenizer menambahkan kata khusus <code>[CLS]</code> di awal dan <code>[SEP]</code> di akhir. Ini karena model dilatih sebelumnya dengan token tersebut, jadi untuk hasil inferensi yang konsisten, token ini juga perlu ditambahkan. Perlu dicatat bahwa tidak semua model menambahkan token khusus, dan ada yang menambahkan token yang berbeda. Beberapa hanya di awal, beberapa hanya di akhir — tokenizer tahu apa yang dibutuhkan oleh masing-masing model dan akan mengaturnya untuk Anda.",Je,H,he,N,ve="Sekarang setelah kita melihat semua langkah individual yang digunakan oleh objek <code>tokenizer</code> saat diterapkan ke teks, mari kita lihat bagaimana ia menangani beberapa urutan (padding!), urutan sangat panjang (truncation!), dan berbagai jenis tensor langsung melalui API utamanya:",Te,Q,we,_,Ie,D,fe;return k=new Pe({props:{fw:Ue[0]}}),g=new Ze({props:{title:"Menggabungkan Semuanya",local:"putting-it-all-together",headingTag:"h1"}}),J=new De({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter2/section6_pt.ipynb"}]}}),w=new d({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQSUwQXNlcXVlbmNlJTIwJTNEJTIwJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTBBJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),U=new d({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2Up",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)`,wrap:!1}}),$=new d({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

model_inputs = tokenizer(sequences)`,wrap:!1}}),V=new d({props:{code:"JTIzJTIwUGFkZGluZyUyMGhpbmdnYSUyMHBhbmphbmclMjB1cnV0YW4lMjB0ZXJwYW5qYW5nJTIwZGFsYW0lMjBiYXRjaCUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEJTIybG9uZ2VzdCUyMiklMEElMEElMjMlMjBQYWRkaW5nJTIwaGluZ2dhJTIwcGFuamFuZyUyMG1ha3NpbXVtJTIwbW9kZWwlMEElMjMlMjAoNTEyJTIwdW50dWslMjBCRVJUJTJGRGlzdGlsQkVSVCklMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRCUyMm1heF9sZW5ndGglMjIpJTBBJTBBJTIzJTIwUGFkZGluZyUyMGhpbmdnYSUyMHBhbmphbmclMjBtYWtzaW11bSUyMHlhbmclMjBkaXRlbnR1a2FuJTBBbW9kZWxfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHNlcXVlbmNlcyUyQyUyMHBhZGRpbmclM0QlMjJtYXhfbGVuZ3RoJTIyJTJDJTIwbWF4X2xlbmd0aCUzRDgp",highlighted:`<span class="hljs-comment"># Padding hingga panjang urutan terpanjang dalam batch</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;longest&quot;</span>)

<span class="hljs-comment"># Padding hingga panjang maksimum model</span>
<span class="hljs-comment"># (512 untuk BERT/DistilBERT)</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>)

<span class="hljs-comment"># Padding hingga panjang maksimum yang ditentukan</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-string">&quot;max_length&quot;</span>, max_length=<span class="hljs-number">8</span>)`,wrap:!1}}),z=new d({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBQb3RvbmclMjB1cnV0YW4lMjB5YW5nJTIwbWVsZWJpaGklMjBwYW5qYW5nJTIwbWFrc2ltdW0lMjBtb2RlbCUwQSUyMyUyMCg1MTIlMjB1bnR1ayUyMEJFUlQlMkZEaXN0aWxCRVJUKSUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjB0cnVuY2F0aW9uJTNEVHJ1ZSklMEElMEElMjMlMjBQb3RvbmclMjB1cnV0YW4lMjB5YW5nJTIwbWVsZWJpaGklMjBwYW5qYW5nJTIwbWFrc2ltdW0lMjB5YW5nJTIwZGl0ZW50dWthbiUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBtYXhfbGVuZ3RoJTNEOCUyQyUyMHRydW5jYXRpb24lM0RUcnVlKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Potong urutan yang melebihi panjang maksimum model</span>
<span class="hljs-comment"># (512 untuk BERT/DistilBERT)</span>
model_inputs = tokenizer(sequences, truncation=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Potong urutan yang melebihi panjang maksimum yang ditentukan</span>
model_inputs = tokenizer(sequences, max_length=<span class="hljs-number">8</span>, truncation=<span class="hljs-literal">True</span>)`,wrap:!1}}),Y=new d({props:{code:"c2VxdWVuY2VzJTIwJTNEJTIwJTVCJTIySSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMEh1Z2dpbmdGYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIyJTJDJTIwJTIyU28lMjBoYXZlJTIwSSElMjIlNUQlMEElMEElMjMlMjBNZW5nZW1iYWxpa2FuJTIwdGVuc29yJTIwUHlUb3JjaCUwQW1vZGVsX2lucHV0cyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBJTIzJTIwTWVuZ2VtYmFsaWthbiUyMGFycmF5JTIwTnVtUHklMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMm5wJTIyKQ==",highlighted:`sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

<span class="hljs-comment"># Mengembalikan tensor PyTorch</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># Mengembalikan array NumPy</span>
model_inputs = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)`,wrap:!1}}),S=new Ze({props:{title:"Token Khusus",local:"special-tokens",headingTag:"h2"}}),G=new d({props:{code:"c2VxdWVuY2UlMjAlM0QlMjAlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoc2VxdWVuY2UpJTBBcHJpbnQobW9kZWxfaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplci50b2tlbml6ZShzZXF1ZW5jZSklMEFpZHMlMjAlM0QlMjB0b2tlbml6ZXIuY29udmVydF90b2tlbnNfdG9faWRzKHRva2VucyklMEFwcmludChpZHMp",highlighted:`sequence = <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>

model_inputs = tokenizer(sequence)
<span class="hljs-built_in">print</span>(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])

tokens = tokenizer.tokenize(sequence)
ids = tokenizer.convert_tokens_to_ids(tokens)
<span class="hljs-built_in">print</span>(ids)`,wrap:!1}}),R=new d({props:{code:"JTVCMTAxJTJDJTIwMTA0NSUyQyUyMDEwMDUlMkMlMjAyMzEwJTJDJTIwMjA0MiUyQyUyMDM0MDMlMkMlMjAyMDA1JTJDJTIwMTAzNyUyQyUyMDE3NjYyJTJDJTIwMTIxNzIlMkMlMjAyNjA3JTJDJTIwMjAyNiUyQyUyMDI4NzglMkMlMjAyMTY2JTJDJTIwMTAxMiUyQyUyMDEwMiU1RCUwQSU1QjEwNDUlMkMlMjAxMDA1JTJDJTIwMjMxMCUyQyUyMDIwNDIlMkMlMjAzNDAzJTJDJTIwMjAwNSUyQyUyMDEwMzclMkMlMjAxNzY2MiUyQyUyMDEyMTcyJTJDJTIwMjYwNyUyQyUyMDIwMjYlMkMlMjAyODc4JTJDJTIwMjE2NiUyQyUyMDEwMTIlNUQ=",highlighted:`[<span class="hljs-number">101</span>, <span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>]
[<span class="hljs-number">1045</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">2310</span>, <span class="hljs-number">2042</span>, <span class="hljs-number">3403</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17662</span>, <span class="hljs-number">12172</span>, <span class="hljs-number">2607</span>, <span class="hljs-number">2026</span>, <span class="hljs-number">2878</span>, <span class="hljs-number">2166</span>, <span class="hljs-number">1012</span>]`,wrap:!1}}),x=new d({props:{code:"cHJpbnQodG9rZW5pemVyLmRlY29kZShtb2RlbF9pbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQpKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUoaWRzKSk=",highlighted:`<span class="hljs-built_in">print</span>(tokenizer.decode(model_inputs[<span class="hljs-string">&quot;input_ids&quot;</span>]))
<span class="hljs-built_in">print</span>(tokenizer.decode(ids))`,wrap:!1}}),C=new d({props:{code:"JTIyJTVCQ0xTJTVEJTIwaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIwJTVCU0VQJTVEJTIyJTBBJTIyaSd2ZSUyMGJlZW4lMjB3YWl0aW5nJTIwZm9yJTIwYSUyMGh1Z2dpbmdmYWNlJTIwY291cnNlJTIwbXklMjB3aG9sZSUyMGxpZmUuJTIy",highlighted:`<span class="hljs-string">&quot;[CLS] i&#x27;ve been waiting for a huggingface course my whole life. [SEP]&quot;</span>
<span class="hljs-string">&quot;i&#x27;ve been waiting for a huggingface course my whole life.&quot;</span>`,wrap:!1}}),H=new Ze({props:{title:"Penutup: Dari Tokenizer ke Model",local:"wrapping-up-from-tokenizer-to-model",headingTag:"h2"}}),Q=new d({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMEElMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQtZmluZXR1bmVkLXNzdC0yLWVuZ2xpc2glMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50KSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbi5mcm9tX3ByZXRyYWluZWQoY2hlY2twb2ludCklMEFzZXF1ZW5jZXMlMjAlM0QlMjAlNUIlMjJJJ3ZlJTIwYmVlbiUyMHdhaXRpbmclMjBmb3IlMjBhJTIwSHVnZ2luZ0ZhY2UlMjBjb3Vyc2UlMjBteSUyMHdob2xlJTIwbGlmZS4lMjIlMkMlMjAlMjJTbyUyMGhhdmUlMjBJISUyMiU1RCUwQSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplcihzZXF1ZW5jZXMlMkMlMjBwYWRkaW5nJTNEVHJ1ZSUyQyUyMHRydW5jYXRpb24lM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFvdXRwdXQlMjAlM0QlMjBtb2RlbCgqKnRva2Vucyk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSequenceClassification

checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [<span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>, <span class="hljs-string">&quot;So have I!&quot;</span>]

tokens = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
output = model(**tokens)`,wrap:!1}}),_=new Le({props:{source:"https://github.com/huggingface/course/blob/main/chapters/id/chapter2/6.mdx"}}),{c(){j=M("meta"),F=t(),y=M("p"),A=t(),i(k.$$.fragment),P=t(),i(g.$$.fragment),L=t(),i(J.$$.fragment),K=t(),h=M("p"),h.textContent=$e,O=t(),T=M("p"),T.innerHTML=We,ee=t(),i(w.$$.fragment),ae=t(),I=M("p"),I.innerHTML=Ve,ne=t(),f=M("p"),f.textContent=Be,le=t(),i(U.$$.fragment),te=t(),Z=M("p"),Z.textContent=ze,se=t(),i($.$$.fragment),ie=t(),W=M("p"),W.textContent=Ee,pe=t(),i(V.$$.fragment),ue=t(),B=M("p"),B.textContent=Ye,me=t(),i(z.$$.fragment),oe=t(),E=M("p"),E.innerHTML=Se,re=t(),i(Y.$$.fragment),Me=t(),i(S.$$.fragment),be=t(),X=M("p"),X.innerHTML=Xe,ce=t(),i(G.$$.fragment),de=t(),i(R.$$.fragment),je=t(),v=M("p"),v.textContent=Ge,ye=t(),i(x.$$.fragment),ke=t(),i(C.$$.fragment),ge=t(),q=M("p"),q.innerHTML=Re,Je=t(),i(H.$$.fragment),he=t(),N=M("p"),N.innerHTML=ve,Te=t(),i(Q.$$.fragment),we=t(),i(_.$$.fragment),Ie=t(),D=M("p"),this.h()},l(e){const a=Fe("svelte-u9bgzb",document.head);j=b(a,"META",{name:!0,content:!0}),a.forEach(n),F=s(e),y=b(e,"P",{}),Ce(y).forEach(n),A=s(e),p(k.$$.fragment,e),P=s(e),p(g.$$.fragment,e),L=s(e),p(J.$$.fragment,e),K=s(e),h=b(e,"P",{"data-svelte-h":!0}),c(h)!=="svelte-1nikemd"&&(h.textContent=$e),O=s(e),T=b(e,"P",{"data-svelte-h":!0}),c(T)!=="svelte-97duk5"&&(T.innerHTML=We),ee=s(e),p(w.$$.fragment,e),ae=s(e),I=b(e,"P",{"data-svelte-h":!0}),c(I)!=="svelte-zxmzbv"&&(I.innerHTML=Ve),ne=s(e),f=b(e,"P",{"data-svelte-h":!0}),c(f)!=="svelte-1jzt4nt"&&(f.textContent=Be),le=s(e),p(U.$$.fragment,e),te=s(e),Z=b(e,"P",{"data-svelte-h":!0}),c(Z)!=="svelte-1ez276c"&&(Z.textContent=ze),se=s(e),p($.$$.fragment,e),ie=s(e),W=b(e,"P",{"data-svelte-h":!0}),c(W)!=="svelte-fylclx"&&(W.textContent=Ee),pe=s(e),p(V.$$.fragment,e),ue=s(e),B=b(e,"P",{"data-svelte-h":!0}),c(B)!=="svelte-115ojl4"&&(B.textContent=Ye),me=s(e),p(z.$$.fragment,e),oe=s(e),E=b(e,"P",{"data-svelte-h":!0}),c(E)!=="svelte-kt37gj"&&(E.innerHTML=Se),re=s(e),p(Y.$$.fragment,e),Me=s(e),p(S.$$.fragment,e),be=s(e),X=b(e,"P",{"data-svelte-h":!0}),c(X)!=="svelte-lj5xkd"&&(X.innerHTML=Xe),ce=s(e),p(G.$$.fragment,e),de=s(e),p(R.$$.fragment,e),je=s(e),v=b(e,"P",{"data-svelte-h":!0}),c(v)!=="svelte-lakep4"&&(v.textContent=Ge),ye=s(e),p(x.$$.fragment,e),ke=s(e),p(C.$$.fragment,e),ge=s(e),q=b(e,"P",{"data-svelte-h":!0}),c(q)!=="svelte-pqjm5s"&&(q.innerHTML=Re),Je=s(e),p(H.$$.fragment,e),he=s(e),N=b(e,"P",{"data-svelte-h":!0}),c(N)!=="svelte-16eukkg"&&(N.innerHTML=ve),Te=s(e),p(Q.$$.fragment,e),we=s(e),p(_.$$.fragment,e),Ie=s(e),D=b(e,"P",{}),Ce(D).forEach(n),this.h()},h(){qe(j,"name","hf:doc:metadata"),qe(j,"content",Oe)},m(e,a){Ae(document.head,j),l(e,F,a),l(e,y,a),l(e,A,a),u(k,e,a),l(e,P,a),u(g,e,a),l(e,L,a),u(J,e,a),l(e,K,a),l(e,h,a),l(e,O,a),l(e,T,a),l(e,ee,a),u(w,e,a),l(e,ae,a),l(e,I,a),l(e,ne,a),l(e,f,a),l(e,le,a),u(U,e,a),l(e,te,a),l(e,Z,a),l(e,se,a),u($,e,a),l(e,ie,a),l(e,W,a),l(e,pe,a),u(V,e,a),l(e,ue,a),l(e,B,a),l(e,me,a),u(z,e,a),l(e,oe,a),l(e,E,a),l(e,re,a),u(Y,e,a),l(e,Me,a),u(S,e,a),l(e,be,a),l(e,X,a),l(e,ce,a),u(G,e,a),l(e,de,a),u(R,e,a),l(e,je,a),l(e,v,a),l(e,ye,a),u(x,e,a),l(e,ke,a),u(C,e,a),l(e,ge,a),l(e,q,a),l(e,Je,a),u(H,e,a),l(e,he,a),l(e,N,a),l(e,Te,a),u(Q,e,a),l(e,we,a),u(_,e,a),l(e,Ie,a),l(e,D,a),fe=!0},p(e,[a]){const xe={};a&1&&(xe.fw=e[0]),k.$set(xe)},i(e){fe||(m(k.$$.fragment,e),m(g.$$.fragment,e),m(J.$$.fragment,e),m(w.$$.fragment,e),m(U.$$.fragment,e),m($.$$.fragment,e),m(V.$$.fragment,e),m(z.$$.fragment,e),m(Y.$$.fragment,e),m(S.$$.fragment,e),m(G.$$.fragment,e),m(R.$$.fragment,e),m(x.$$.fragment,e),m(C.$$.fragment,e),m(H.$$.fragment,e),m(Q.$$.fragment,e),m(_.$$.fragment,e),fe=!0)},o(e){o(k.$$.fragment,e),o(g.$$.fragment,e),o(J.$$.fragment,e),o(w.$$.fragment,e),o(U.$$.fragment,e),o($.$$.fragment,e),o(V.$$.fragment,e),o(z.$$.fragment,e),o(Y.$$.fragment,e),o(S.$$.fragment,e),o(G.$$.fragment,e),o(R.$$.fragment,e),o(x.$$.fragment,e),o(C.$$.fragment,e),o(H.$$.fragment,e),o(Q.$$.fragment,e),o(_.$$.fragment,e),fe=!1},d(e){e&&(n(F),n(y),n(A),n(P),n(L),n(K),n(h),n(O),n(T),n(ee),n(ae),n(I),n(ne),n(f),n(le),n(te),n(Z),n(se),n(ie),n(W),n(pe),n(ue),n(B),n(me),n(oe),n(E),n(re),n(Me),n(be),n(X),n(ce),n(de),n(je),n(v),n(ye),n(ke),n(ge),n(q),n(Je),n(he),n(N),n(Te),n(we),n(Ie),n(D)),n(j),r(k,e),r(g,e),r(J,e),r(w,e),r(U,e),r($,e),r(V,e),r(z,e),r(Y,e),r(S,e),r(G,e),r(R,e),r(x,e),r(C,e),r(H,e),r(Q,e),r(_,e)}}}const Oe='{"title":"Menggabungkan Semuanya","local":"putting-it-all-together","sections":[{"title":"Token Khusus","local":"special-tokens","sections":[],"depth":2},{"title":"Penutup: Dari Tokenizer ke Model","local":"wrapping-up-from-tokenizer-to-model","sections":[],"depth":2}],"depth":1}';function ea(Ue,j,F){let y="pt";return Ne(()=>{const A=new URLSearchParams(window.location.search);F(0,y=A.get("fw")||"pt")}),[y]}class pa extends Qe{constructor(j){super(),_e(this,j,ea,Ke,He,{})}}export{pa as component};
