import{s as Js,n as xs,o as ks}from"../chunks/scheduler.bdbef820.js";import{S as Cs,i as Us,g as p,s as l,r as h,A as Ws,h as r,f as n,c as t,j as Ts,u as c,x as m,k as _s,y as Gs,a as e,v as u,d as i,t as b,w as j}from"../chunks/index.c0aea24a.js";import{C as V}from"../chunks/CodeBlock.6ccca92e.js";import{H as cs,E as Is}from"../chunks/EditOnGithub.725ee0c1.js";function vs(us){let o,Y,Z,N,d,F,f,is="This guide shows specific methods for processing text datasets. Learn how to:",P,g,bs='<li>Tokenize a dataset with <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset.map">map()</a>.</li> <li>Align dataset labels with label ids for NLI datasets.</li>',R,y,js='For a guide on how to process any type of dataset, take a look at the <a class="underline decoration-sky-400 decoration-2 font-semibold" href="./process">general process guide</a>.',A,$,E,w,os='The <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset.map">map()</a> function supports processing batches of examples at once which speeds up tokenization.',B,M,ds='Load a tokenizer from 🤗 <a href="https://huggingface.co/transformers/" rel="nofollow">Transformers</a>:',Q,T,X,_,fs='Set the <code>batched</code> parameter to <code>True</code> in the <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset.map">map()</a> function to apply the tokenizer to batches of examples:',S,J,D,x,gs='The <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset.map">map()</a> function converts the returned values to a PyArrow-supported format. But explicitly returning the tensors as NumPy arrays is faster because it is a natively supported PyArrow format. Set <code>return_tensors=&quot;np&quot;</code> when you tokenize your text:',K,k,O,C,ss,U,ys='The <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset.align_labels_with_mapping">align_labels_with_mapping()</a> function aligns a dataset label id with the label name. Not all 🤗 Transformers models follow the prescribed label mapping of the original dataset, especially for NLI datasets. For example, the <a href="https://huggingface.co/datasets/glue" rel="nofollow">MNLI</a> dataset uses the following label mapping:',as,W,ns,G,$s="To align the dataset label mapping with the mapping used by a model, create a dictionary of the label name and id to align on:",es,I,ls,v,ws='Pass the dictionary of the label mappings to the <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset.align_labels_with_mapping">align_labels_with_mapping()</a> function, and the column to align on:',ts,q,ps,z,Ms="You can also use this function to assign a custom mapping of labels to ids.",rs,H,ms,L,hs;return d=new cs({props:{title:"Process text data",local:"process-text-data",headingTag:"h1"}}),$=new cs({props:{title:"Map",local:"map",headingTag:"h2"}}),T=new V({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJiZXJ0LWJhc2UtY2FzZWQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`,wrap:!1}}),J=new V({props:{code:"ZGF0YXNldCUyMCUzRCUyMGRhdGFzZXQubWFwKGxhbWJkYSUyMGV4YW1wbGVzJTNBJTIwdG9rZW5pemVyKGV4YW1wbGVzJTVCJTIydGV4dCUyMiU1RCklMkMlMjBiYXRjaGVkJTNEVHJ1ZSklMEFkYXRhc2V0JTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.<span class="hljs-built_in">map</span>(<span class="hljs-keyword">lambda</span> examples: tokenizer(examples[<span class="hljs-string">&quot;text&quot;</span>]), batched=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;the rock is destined to be the 21st century\\&#x27;s new &quot; conan &quot; and that he\\&#x27;s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .&#x27;</span>, 
 <span class="hljs-string">&#x27;label&#x27;</span>: <span class="hljs-number">1</span>, 
 <span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">2600</span>, <span class="hljs-number">2003</span>, <span class="hljs-number">16036</span>, <span class="hljs-number">2000</span>, <span class="hljs-number">2022</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">7398</span>, <span class="hljs-number">2301</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">1055</span>, <span class="hljs-number">2047</span>, <span class="hljs-number">1000</span>, <span class="hljs-number">16608</span>, <span class="hljs-number">1000</span>, <span class="hljs-number">1998</span>, <span class="hljs-number">2008</span>, <span class="hljs-number">2002</span>, <span class="hljs-number">1005</span>, <span class="hljs-number">1055</span>, <span class="hljs-number">2183</span>, <span class="hljs-number">2000</span>, <span class="hljs-number">2191</span>, <span class="hljs-number">1037</span>, <span class="hljs-number">17624</span>, <span class="hljs-number">2130</span>, <span class="hljs-number">3618</span>, <span class="hljs-number">2084</span>, <span class="hljs-number">7779</span>, <span class="hljs-number">29058</span>, <span class="hljs-number">8625</span>, <span class="hljs-number">13327</span>, <span class="hljs-number">1010</span>, <span class="hljs-number">3744</span>, <span class="hljs-number">1011</span>, <span class="hljs-number">18856</span>, <span class="hljs-number">19513</span>, <span class="hljs-number">3158</span>, <span class="hljs-number">5477</span>, <span class="hljs-number">4168</span>, <span class="hljs-number">2030</span>, <span class="hljs-number">7112</span>, <span class="hljs-number">16562</span>, <span class="hljs-number">2140</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`,wrap:!1}}),k=new V({props:{code:"ZGF0YXNldCUyMCUzRCUyMGRhdGFzZXQubWFwKGxhbWJkYSUyMGV4YW1wbGVzJTNBJTIwdG9rZW5pemVyKGV4YW1wbGVzJTVCJTIydGV4dCUyMiU1RCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIybnAlMjIpJTJDJTIwYmF0Y2hlZCUzRFRydWUp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.<span class="hljs-built_in">map</span>(<span class="hljs-keyword">lambda</span> examples: tokenizer(examples[<span class="hljs-string">&quot;text&quot;</span>], return_tensors=<span class="hljs-string">&quot;np&quot;</span>), batched=<span class="hljs-literal">True</span>)',wrap:!1}}),C=new cs({props:{title:"Align",local:"align",headingTag:"h2"}}),W=new V({props:{code:"bGFiZWwyaWQlMjAlM0QlMjAlN0IlMjJlbnRhaWxtZW50JTIyJTNBJTIwMCUyQyUyMCUyMm5ldXRyYWwlMjIlM0ElMjAxJTJDJTIwJTIyY29udHJhZGljdGlvbiUyMiUzQSUyMDIlN0Q=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>label2id = {<span class="hljs-string">&quot;entailment&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;neutral&quot;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&quot;contradiction&quot;</span>: <span class="hljs-number">2</span>}',wrap:!1}}),I=new V({props:{code:"bGFiZWwyaWQlMjAlM0QlMjAlN0IlMjJjb250cmFkaWN0aW9uJTIyJTNBJTIwMCUyQyUyMCUyMm5ldXRyYWwlMjIlM0ElMjAxJTJDJTIwJTIyZW50YWlsbWVudCUyMiUzQSUyMDIlN0Q=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>label2id = {<span class="hljs-string">&quot;contradiction&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;neutral&quot;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&quot;entailment&quot;</span>: <span class="hljs-number">2</span>}',wrap:!1}}),q=new V({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBbW5saSUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJnbHVlJTIyJTJDJTIwJTIybW5saSUyMiUyQyUyMHNwbGl0JTNEJTIydHJhaW4lMjIpJTBBbW5saV9hbGlnbmVkJTIwJTNEJTIwbW5saS5hbGlnbl9sYWJlbHNfd2l0aF9tYXBwaW5nKGxhYmVsMmlkJTJDJTIwJTIybGFiZWwlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>mnli = load_dataset(<span class="hljs-string">&quot;glue&quot;</span>, <span class="hljs-string">&quot;mnli&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>mnli_aligned = mnli.align_labels_with_mapping(label2id, <span class="hljs-string">&quot;label&quot;</span>)`,wrap:!1}}),H=new Is({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/nlp_process.mdx"}}),{c(){o=p("meta"),Y=l(),Z=p("p"),N=l(),h(d.$$.fragment),F=l(),f=p("p"),f.textContent=is,P=l(),g=p("ul"),g.innerHTML=bs,R=l(),y=p("p"),y.innerHTML=js,A=l(),h($.$$.fragment),E=l(),w=p("p"),w.innerHTML=os,B=l(),M=p("p"),M.innerHTML=ds,Q=l(),h(T.$$.fragment),X=l(),_=p("p"),_.innerHTML=fs,S=l(),h(J.$$.fragment),D=l(),x=p("p"),x.innerHTML=gs,K=l(),h(k.$$.fragment),O=l(),h(C.$$.fragment),ss=l(),U=p("p"),U.innerHTML=ys,as=l(),h(W.$$.fragment),ns=l(),G=p("p"),G.textContent=$s,es=l(),h(I.$$.fragment),ls=l(),v=p("p"),v.innerHTML=ws,ts=l(),h(q.$$.fragment),ps=l(),z=p("p"),z.textContent=Ms,rs=l(),h(H.$$.fragment),ms=l(),L=p("p"),this.h()},l(s){const a=Ws("svelte-u9bgzb",document.head);o=r(a,"META",{name:!0,content:!0}),a.forEach(n),Y=t(s),Z=r(s,"P",{}),Ts(Z).forEach(n),N=t(s),c(d.$$.fragment,s),F=t(s),f=r(s,"P",{"data-svelte-h":!0}),m(f)!=="svelte-zxifil"&&(f.textContent=is),P=t(s),g=r(s,"UL",{"data-svelte-h":!0}),m(g)!=="svelte-sss7d5"&&(g.innerHTML=bs),R=t(s),y=r(s,"P",{"data-svelte-h":!0}),m(y)!=="svelte-3s2bzp"&&(y.innerHTML=js),A=t(s),c($.$$.fragment,s),E=t(s),w=r(s,"P",{"data-svelte-h":!0}),m(w)!=="svelte-1oefonr"&&(w.innerHTML=os),B=t(s),M=r(s,"P",{"data-svelte-h":!0}),m(M)!=="svelte-b5bjp1"&&(M.innerHTML=ds),Q=t(s),c(T.$$.fragment,s),X=t(s),_=r(s,"P",{"data-svelte-h":!0}),m(_)!=="svelte-4q53zq"&&(_.innerHTML=fs),S=t(s),c(J.$$.fragment,s),D=t(s),x=r(s,"P",{"data-svelte-h":!0}),m(x)!=="svelte-ncdkwm"&&(x.innerHTML=gs),K=t(s),c(k.$$.fragment,s),O=t(s),c(C.$$.fragment,s),ss=t(s),U=r(s,"P",{"data-svelte-h":!0}),m(U)!=="svelte-awxet0"&&(U.innerHTML=ys),as=t(s),c(W.$$.fragment,s),ns=t(s),G=r(s,"P",{"data-svelte-h":!0}),m(G)!=="svelte-tn6t6n"&&(G.textContent=$s),es=t(s),c(I.$$.fragment,s),ls=t(s),v=r(s,"P",{"data-svelte-h":!0}),m(v)!=="svelte-5onshg"&&(v.innerHTML=ws),ts=t(s),c(q.$$.fragment,s),ps=t(s),z=r(s,"P",{"data-svelte-h":!0}),m(z)!=="svelte-18y4a1w"&&(z.textContent=Ms),rs=t(s),c(H.$$.fragment,s),ms=t(s),L=r(s,"P",{}),Ts(L).forEach(n),this.h()},h(){_s(o,"name","hf:doc:metadata"),_s(o,"content",qs)},m(s,a){Gs(document.head,o),e(s,Y,a),e(s,Z,a),e(s,N,a),u(d,s,a),e(s,F,a),e(s,f,a),e(s,P,a),e(s,g,a),e(s,R,a),e(s,y,a),e(s,A,a),u($,s,a),e(s,E,a),e(s,w,a),e(s,B,a),e(s,M,a),e(s,Q,a),u(T,s,a),e(s,X,a),e(s,_,a),e(s,S,a),u(J,s,a),e(s,D,a),e(s,x,a),e(s,K,a),u(k,s,a),e(s,O,a),u(C,s,a),e(s,ss,a),e(s,U,a),e(s,as,a),u(W,s,a),e(s,ns,a),e(s,G,a),e(s,es,a),u(I,s,a),e(s,ls,a),e(s,v,a),e(s,ts,a),u(q,s,a),e(s,ps,a),e(s,z,a),e(s,rs,a),u(H,s,a),e(s,ms,a),e(s,L,a),hs=!0},p:xs,i(s){hs||(i(d.$$.fragment,s),i($.$$.fragment,s),i(T.$$.fragment,s),i(J.$$.fragment,s),i(k.$$.fragment,s),i(C.$$.fragment,s),i(W.$$.fragment,s),i(I.$$.fragment,s),i(q.$$.fragment,s),i(H.$$.fragment,s),hs=!0)},o(s){b(d.$$.fragment,s),b($.$$.fragment,s),b(T.$$.fragment,s),b(J.$$.fragment,s),b(k.$$.fragment,s),b(C.$$.fragment,s),b(W.$$.fragment,s),b(I.$$.fragment,s),b(q.$$.fragment,s),b(H.$$.fragment,s),hs=!1},d(s){s&&(n(Y),n(Z),n(N),n(F),n(f),n(P),n(g),n(R),n(y),n(A),n(E),n(w),n(B),n(M),n(Q),n(X),n(_),n(S),n(D),n(x),n(K),n(O),n(ss),n(U),n(as),n(ns),n(G),n(es),n(ls),n(v),n(ts),n(ps),n(z),n(rs),n(ms),n(L)),n(o),j(d,s),j($,s),j(T,s),j(J,s),j(k,s),j(C,s),j(W,s),j(I,s),j(q,s),j(H,s)}}}const qs='{"title":"Process text data","local":"process-text-data","sections":[{"title":"Map","local":"map","sections":[],"depth":2},{"title":"Align","local":"align","sections":[],"depth":2}],"depth":1}';function zs(us){return ks(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ys extends Cs{constructor(o){super(),Us(this,o,zs,vs,Js,{})}}export{Ys as component};
