import{s as et,n as at,o as st}from"../chunks/scheduler.d6170356.js";import{S as lt,i as nt,g as d,s as l,r as G,A as it,h as r,f as a,c as n,j as O,u as Z,x as U,k as tt,y as dt,a as s,v as _,d as R,t as T,w as X}from"../chunks/index.fcd4cc08.js";import{C as N}from"../chunks/CodeBlock.7b16bdef.js";import{H as rt,E as ot}from"../chunks/EditOnGithub.da2b595c.js";function ht(L){let i,$,b,j,o,Y,h,S=`<a href="https://github.com/dask/dask" rel="nofollow">Dask</a> is a parallel and distributed computing library that scales the existing Python and PyData ecosystem.
Since it uses <a href="https://filesystem-spec.readthedocs.io" rel="nofollow">fsspec</a> to read and write remote data, you can use the Hugging Face paths (<a href="https://huggingface.co/docs/huggingface_hub/guides/hf_file_system#integrations" rel="nofollow"><code>hf://</code></a>) to read and write data on the Hub:`,B,p,x='First you need to <a href="../huggingface_hub/quick-start#login">Login with your Hugging Face account</a>, for example using:',F,c,q,m,P='Then you can <a href="../huggingface_hub/quick-start#create-a-repository">Create a dataset repository</a>, for example using:',H,f,V,u,D='Finally, you can use <a href="https://huggingface.co/docs/huggingface_hub/guides/hf_file_system#integrations" rel="nofollow">Hugging Face paths</a> in Dask:',z,y,C,g,A=`This creates a dataset repository <code>username/my_dataset</code> containing your Dask dataset in Parquet format.
You can reload it later:`,v,M,W,J,K='For more information on the Hugging Face paths and how they are implemented, please refer to the <a href="https://huggingface.co/docs/huggingface_hub/guides/hf_file_system" rel="nofollow">the client library’s documentation on the HfFileSystem</a>.',I,k,Q,w,E;return o=new rt({props:{title:"Dask",local:"dask",headingTag:"h1"}}),c=new N({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:'huggingface-<span class="hljs-keyword">cli</span> login',wrap:!1}}),f=new N({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEhmQXBpJTBBJTBBSGZBcGkoKS5jcmVhdGVfcmVwbyhyZXBvX2lkJTNEJTIydXNlcm5hbWUlMkZteV9kYXRhc2V0JTIyJTJDJTIwcmVwb190eXBlJTNEJTIyZGF0YXNldCUyMik=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi

HfApi().create_repo(repo_id=<span class="hljs-string">&quot;username/my_dataset&quot;</span>, repo_type=<span class="hljs-string">&quot;dataset&quot;</span>)`,wrap:!1}}),y=new N({props:{code:"aW1wb3J0JTIwZGFzay5kYXRhZnJhbWUlMjBhcyUyMGRkJTBBJTBBZGYudG9fcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTIyKSUwQSUwQSUyMyUyMG9yJTIwd3JpdGUlMjBpbiUyMHNlcGFyYXRlJTIwZGlyZWN0b3JpZXMlMjBpZiUyMHRoZSUyMGRhdGFzZXQlMjBoYXMlMjB0cmFpbiUyRnZhbGlkYXRpb24lMkZ0ZXN0JTIwc3BsaXRzJTBBZGZfdHJhaW4udG9fcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdHJhaW4lMjIpJTBBZGZfdmFsaWQudG9fcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdmFsaWRhdGlvbiUyMiklMEFkZl90ZXN0JTIwLnRvX3BhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGbXlfZGF0YXNldCUyRnRlc3QlMjIp",highlighted:`<span class="hljs-keyword">import</span> dask.dataframe <span class="hljs-keyword">as</span> dd

df.to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset&quot;</span>)

<span class="hljs-comment"># or write in separate directories if the dataset has train/validation/test splits</span>
df_train.to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/train&quot;</span>)
df_valid.to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/validation&quot;</span>)
df_test .to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/test&quot;</span>)`,wrap:!1}}),M=new N({props:{code:"aW1wb3J0JTIwZGFzay5kYXRhZnJhbWUlMjBhcyUyMGRkJTBBJTBBZGYlMjAlM0QlMjBkZC5yZWFkX3BhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGbXlfZGF0YXNldCUyMiklMEElMEElMjMlMjBvciUyMHJlYWQlMjBmcm9tJTIwc2VwYXJhdGUlMjBkaXJlY3RvcmllcyUyMGlmJTIwdGhlJTIwZGF0YXNldCUyMGhhcyUyMHRyYWluJTJGdmFsaWRhdGlvbiUyRnRlc3QlMjBzcGxpdHMlMEFkZl90cmFpbiUyMCUzRCUyMGRkLnJlYWRfcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdHJhaW4lMjIpJTBBZGZfdmFsaWQlMjAlM0QlMjBkZC5yZWFkX3BhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGbXlfZGF0YXNldCUyRnZhbGlkYXRpb24lMjIpJTBBZGZfdGVzdCUyMCUyMCUzRCUyMGRkLnJlYWRfcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdGVzdCUyMik=",highlighted:`<span class="hljs-keyword">import</span> dask.dataframe <span class="hljs-keyword">as</span> dd

df = dd.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset&quot;</span>)

<span class="hljs-comment"># or read from separate directories if the dataset has train/validation/test splits</span>
df_train = dd.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/train&quot;</span>)
df_valid = dd.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/validation&quot;</span>)
df_test  = dd.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/test&quot;</span>)`,wrap:!1}}),k=new ot({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-dask.md"}}),{c(){i=d("meta"),$=l(),b=d("p"),j=l(),G(o.$$.fragment),Y=l(),h=d("p"),h.innerHTML=S,B=l(),p=d("p"),p.innerHTML=x,F=l(),G(c.$$.fragment),q=l(),m=d("p"),m.innerHTML=P,H=l(),G(f.$$.fragment),V=l(),u=d("p"),u.innerHTML=D,z=l(),G(y.$$.fragment),C=l(),g=d("p"),g.innerHTML=A,v=l(),G(M.$$.fragment),W=l(),J=d("p"),J.innerHTML=K,I=l(),G(k.$$.fragment),Q=l(),w=d("p"),this.h()},l(t){const e=it("svelte-u9bgzb",document.head);i=r(e,"META",{name:!0,content:!0}),e.forEach(a),$=n(t),b=r(t,"P",{}),O(b).forEach(a),j=n(t),Z(o.$$.fragment,t),Y=n(t),h=r(t,"P",{"data-svelte-h":!0}),U(h)!=="svelte-1nkggm5"&&(h.innerHTML=S),B=n(t),p=r(t,"P",{"data-svelte-h":!0}),U(p)!=="svelte-1hbzj8b"&&(p.innerHTML=x),F=n(t),Z(c.$$.fragment,t),q=n(t),m=r(t,"P",{"data-svelte-h":!0}),U(m)!=="svelte-1tu70ef"&&(m.innerHTML=P),H=n(t),Z(f.$$.fragment,t),V=n(t),u=r(t,"P",{"data-svelte-h":!0}),U(u)!=="svelte-qq5b5u"&&(u.innerHTML=D),z=n(t),Z(y.$$.fragment,t),C=n(t),g=r(t,"P",{"data-svelte-h":!0}),U(g)!=="svelte-14rczfs"&&(g.innerHTML=A),v=n(t),Z(M.$$.fragment,t),W=n(t),J=r(t,"P",{"data-svelte-h":!0}),U(J)!=="svelte-1k6ibc2"&&(J.innerHTML=K),I=n(t),Z(k.$$.fragment,t),Q=n(t),w=r(t,"P",{}),O(w).forEach(a),this.h()},h(){tt(i,"name","hf:doc:metadata"),tt(i,"content",pt)},m(t,e){dt(document.head,i),s(t,$,e),s(t,b,e),s(t,j,e),_(o,t,e),s(t,Y,e),s(t,h,e),s(t,B,e),s(t,p,e),s(t,F,e),_(c,t,e),s(t,q,e),s(t,m,e),s(t,H,e),_(f,t,e),s(t,V,e),s(t,u,e),s(t,z,e),_(y,t,e),s(t,C,e),s(t,g,e),s(t,v,e),_(M,t,e),s(t,W,e),s(t,J,e),s(t,I,e),_(k,t,e),s(t,Q,e),s(t,w,e),E=!0},p:at,i(t){E||(R(o.$$.fragment,t),R(c.$$.fragment,t),R(f.$$.fragment,t),R(y.$$.fragment,t),R(M.$$.fragment,t),R(k.$$.fragment,t),E=!0)},o(t){T(o.$$.fragment,t),T(c.$$.fragment,t),T(f.$$.fragment,t),T(y.$$.fragment,t),T(M.$$.fragment,t),T(k.$$.fragment,t),E=!1},d(t){t&&(a($),a(b),a(j),a(Y),a(h),a(B),a(p),a(F),a(q),a(m),a(H),a(V),a(u),a(z),a(C),a(g),a(v),a(W),a(J),a(I),a(Q),a(w)),a(i),X(o,t),X(c,t),X(f,t),X(y,t),X(M,t),X(k,t)}}}const pt='{"title":"Dask","local":"dask","sections":[],"depth":1}';function ct(L){return st(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class gt extends lt{constructor(i){super(),nt(this,i,ct,ht,et,{})}}export{gt as component};
