import{s as ja,o as wa,n as Ua}from"../chunks/scheduler.bdbef820.js";import{S as ka,i as Ia,g as r,s as n,r as c,A as $a,h as i,f as t,c as l,j as Ja,u as h,x as o,k as Ta,y as ba,a as e,v as u,d,t as f,w as y}from"../chunks/index.c0aea24a.js";import{T as _a}from"../chunks/Tip.31005f7d.js";import{C as ra}from"../chunks/CodeBlock.6ccca92e.js";import{H as pa,E as Ca}from"../chunks/EditOnGithub.725ee0c1.js";function Ra(W){let p,M='In a different session, a Spark DataFrame doesn’t have the same <a href="https://spark.apache.org/docs/3.2.0/api/python/reference/api/pyspark.sql.DataFrame.semanticHash.html" rel="nofollow">semantic hash</a>, and it will rerun a Spark job and store it in a new cache.';return{c(){p=r("p"),p.innerHTML=M},l(m){p=i(m,"P",{"data-svelte-h":!0}),o(p)!=="svelte-1m6gjz4"&&(p.innerHTML=M)},m(m,Y){e(m,p,Y)},p:Ua,d(m){m&&t(p)}}}function Fa(W){let p,M,m,Y,J,S,T,ia='This document is a quick introduction to using 🤗 Datasets with Spark, with a particular focus on how to load a Spark DataFrame into a <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset">Dataset</a> object.',z,j,ma="From there, you have fast access to any element and you can use it as a data loader to train models.",X,w,x,U,oa=`A <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset">Dataset</a> object is a wrapper of an Arrow table, which allows fast reads from arrays in the dataset to PyTorch, TensorFlow and JAX tensors.
The Arrow table is memory mapped from disk, which can load datasets bigger than your available RAM.`,B,k,ca='You can get a <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset">Dataset</a> from a Spark DataFrame using <code>Dataset.from_spark()</code>:',E,I,H,$,ha='The Spark workers write the dataset on disk in a cache directory as Arrow files, and the <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset">Dataset</a> is loaded from there.',N,b,ua='Alternatively, you can skip materialization by using <code>IterableDataset.from_spark()</code>, which returns an <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a>:',L,_,V,C,A,R,da=`When using <code>Dataset.from_spark()</code>, the resulting <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset">Dataset</a> is cached; if you call <code>Dataset.from_spark()</code> multiple
times on the same DataFrame it won’t re-run the Spark job that writes the dataset as Arrow files on disk.`,P,F,fa=`You can set the cache location by passing <code>cache_dir=</code> to <code>Dataset.from_spark()</code>.
Make sure to use a disk that is available to both your workers and your current machine (the driver).`,K,g,O,D,aa,Q,ya=`If your dataset is made of images, audio data or N-dimensional arrays, you can specify the <code>features=</code> argument in
<code>Dataset.from_spark()</code> (or <code>IterableDataset.from_spark()</code>):`,sa,G,ta,q,ga='You can check the <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Features">Features</a> documentation to know about all the feature types available.',ea,Z,na,v,la;return J=new pa({props:{title:"Use with Spark",local:"use-with-spark",headingTag:"h1"}}),w=new pa({props:{title:"Load from Spark",local:"load-from-spark",headingTag:"h2"}}),I=new ra({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQWRmJTIwJTNEJTIwc3BhcmsuY3JlYXRlRGF0YUZyYW1lKCUwQSUyMCUyMCUyMCUyMGRhdGElM0QlNUIlNUIxJTJDJTIwJTIyRWxpYSUyMiU1RCUyQyUyMCU1QjIlMkMlMjAlMjJUZW8lMjIlNUQlMkMlMjAlNUIzJTJDJTIwJTIyRmFuZyUyMiU1RCU1RCUyQyUwQSUyMCUyMCUyMCUyMGNvbHVtbnMlM0QlNUIlMjJpZCUyMiUyQyUyMCUyMm5hbWUlMjIlNUQlMkMlMEEpJTBBZHMlMjAlM0QlMjBEYXRhc2V0LmZyb21fc3BhcmsoZGYp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>df = spark.createDataFrame(
<span class="hljs-meta">... </span>    data=[[<span class="hljs-number">1</span>, <span class="hljs-string">&quot;Elia&quot;</span>], [<span class="hljs-number">2</span>, <span class="hljs-string">&quot;Teo&quot;</span>], [<span class="hljs-number">3</span>, <span class="hljs-string">&quot;Fang&quot;</span>]],
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;id&quot;</span>, <span class="hljs-string">&quot;name&quot;</span>],
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_spark(df)`,wrap:!1}}),_=new ra({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwSXRlcmFibGVEYXRhc2V0JTBBZGYlMjAlM0QlMjBzcGFyay5jcmVhdGVEYXRhRnJhbWUoJTBBJTIwJTIwJTIwJTIwZGF0YSUzRCU1QiU1QjElMkMlMjAlMjJFbGlhJTIyJTVEJTJDJTIwJTVCMiUyQyUyMCUyMlRlbyUyMiU1RCUyQyUyMCU1QjMlMkMlMjAlMjJGYW5nJTIyJTVEJTVEJTJDJTBBJTIwJTIwJTIwJTIwY29sdW1ucyUzRCU1QiUyMmlkJTIyJTJDJTIwJTIybmFtZSUyMiU1RCUyQyUwQSklMEFkcyUyMCUzRCUyMEl0ZXJhYmxlRGF0YXNldC5mcm9tX3NwYXJrKGRmKSUwQXByaW50KG5leHQoaXRlcihkcykpKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> IterableDataset
<span class="hljs-meta">&gt;&gt;&gt; </span>df = spark.createDataFrame(
<span class="hljs-meta">... </span>    data=[[<span class="hljs-number">1</span>, <span class="hljs-string">&quot;Elia&quot;</span>], [<span class="hljs-number">2</span>, <span class="hljs-string">&quot;Teo&quot;</span>], [<span class="hljs-number">3</span>, <span class="hljs-string">&quot;Fang&quot;</span>]],
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;id&quot;</span>, <span class="hljs-string">&quot;name&quot;</span>],
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = IterableDataset.from_spark(df)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(ds)))
{<span class="hljs-string">&quot;id&quot;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&quot;name&quot;</span>: <span class="hljs-string">&quot;Elia&quot;</span>}`,wrap:!1}}),C=new pa({props:{title:"Caching",local:"caching",headingTag:"h3"}}),g=new _a({props:{warning:!0,$$slots:{default:[Ra]},$$scope:{ctx:W}}}),D=new pa({props:{title:"Feature types",local:"feature-types",headingTag:"h3"}}),G=new ra({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUyQyUyMEZlYXR1cmVzJTJDJTIwSW1hZ2UlMkMlMjBWYWx1ZSUwQWRhdGElMjAlM0QlMjAlNUIoMCUyQyUyMG9wZW4oJTIyaW1hZ2UucG5nJTIyJTJDJTIwJTIycmIlMjIpLnJlYWQoKSklNUQlMEFkZiUyMCUzRCUyMHNwYXJrLmNyZWF0ZURhdGFGcmFtZShkYXRhJTJDJTIwJTIyaWR4JTNBJTIwaW50JTJDJTIwaW1hZ2UlM0ElMjBiaW5hcnklMjIpJTBBJTIzJTIwQWxzbyUyMHdvcmtzJTIwaWYlMjB5b3UlMjBoYXZlJTIwYXJyYXlzJTBBJTIzJTIwZGF0YSUyMCUzRCUyMCU1QigwJTJDJTIwbnAuemVyb3Moc2hhcGUlM0QoMzIlMkMlMjAzMiUyQyUyMDMpJTJDJTIwZHR5cGUlM0RucC5pbnQzMikudG9saXN0KCkpJTVEJTBBJTIzJTIwZGYlMjAlM0QlMjBzcGFyay5jcmVhdGVEYXRhRnJhbWUoZGF0YSUyQyUyMCUyMmlkeCUzQSUyMGludCUyQyUyMGltYWdlJTNBJTIwYXJyYXklM0NhcnJheSUzQ2FycmF5JTNDaW50JTNFJTNFJTNFJTIyKSUwQWZlYXR1cmVzJTIwJTNEJTIwRmVhdHVyZXMoJTdCJTIyaWR4JTIyJTNBJTIwVmFsdWUoJTIyaW50NjQlMjIpJTJDJTIwJTIyaW1hZ2UlMjIlM0ElMjBJbWFnZSgpJTdEKSUwQWRhdGFzZXQlMjAlM0QlMjBEYXRhc2V0LmZyb21fc3BhcmsoZGYlMkMlMjBmZWF0dXJlcyUzRGZlYXR1cmVzKSUwQWRhdGFzZXQlNUIwJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Image, Value
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [(<span class="hljs-number">0</span>, <span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;image.png&quot;</span>, <span class="hljs-string">&quot;rb&quot;</span>).read())]
<span class="hljs-meta">&gt;&gt;&gt; </span>df = spark.createDataFrame(data, <span class="hljs-string">&quot;idx: int, image: binary&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Also works if you have arrays</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># data = [(0, np.zeros(shape=(32, 32, 3), dtype=np.int32).tolist())]</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># df = spark.createDataFrame(data, &quot;idx: int, image: array&lt;array&lt;array&lt;int&gt;&gt;&gt;&quot;)</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;idx&quot;</span>: Value(<span class="hljs-string">&quot;int64&quot;</span>), <span class="hljs-string">&quot;image&quot;</span>: Image()})
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = Dataset.from_spark(df, features=features)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;idx&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;image&#x27;</span>: &lt;PIL.PngImagePlugin.PngImageFile image mode=RGB size=32x32&gt;}`,wrap:!1}}),Z=new Ca({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/use_with_spark.mdx"}}),{c(){p=r("meta"),M=n(),m=r("p"),Y=n(),c(J.$$.fragment),S=n(),T=r("p"),T.innerHTML=ia,z=n(),j=r("p"),j.textContent=ma,X=n(),c(w.$$.fragment),x=n(),U=r("p"),U.innerHTML=oa,B=n(),k=r("p"),k.innerHTML=ca,E=n(),c(I.$$.fragment),H=n(),$=r("p"),$.innerHTML=ha,N=n(),b=r("p"),b.innerHTML=ua,L=n(),c(_.$$.fragment),V=n(),c(C.$$.fragment),A=n(),R=r("p"),R.innerHTML=da,P=n(),F=r("p"),F.innerHTML=fa,K=n(),c(g.$$.fragment),O=n(),c(D.$$.fragment),aa=n(),Q=r("p"),Q.innerHTML=ya,sa=n(),c(G.$$.fragment),ta=n(),q=r("p"),q.innerHTML=ga,ea=n(),c(Z.$$.fragment),na=n(),v=r("p"),this.h()},l(a){const s=$a("svelte-u9bgzb",document.head);p=i(s,"META",{name:!0,content:!0}),s.forEach(t),M=l(a),m=i(a,"P",{}),Ja(m).forEach(t),Y=l(a),h(J.$$.fragment,a),S=l(a),T=i(a,"P",{"data-svelte-h":!0}),o(T)!=="svelte-hb9bfd"&&(T.innerHTML=ia),z=l(a),j=i(a,"P",{"data-svelte-h":!0}),o(j)!=="svelte-1pvlf1d"&&(j.textContent=ma),X=l(a),h(w.$$.fragment,a),x=l(a),U=i(a,"P",{"data-svelte-h":!0}),o(U)!=="svelte-hoaun3"&&(U.innerHTML=oa),B=l(a),k=i(a,"P",{"data-svelte-h":!0}),o(k)!=="svelte-1tsops2"&&(k.innerHTML=ca),E=l(a),h(I.$$.fragment,a),H=l(a),$=i(a,"P",{"data-svelte-h":!0}),o($)!=="svelte-176j43u"&&($.innerHTML=ha),N=l(a),b=i(a,"P",{"data-svelte-h":!0}),o(b)!=="svelte-110deyg"&&(b.innerHTML=ua),L=l(a),h(_.$$.fragment,a),V=l(a),h(C.$$.fragment,a),A=l(a),R=i(a,"P",{"data-svelte-h":!0}),o(R)!=="svelte-1bp9c47"&&(R.innerHTML=da),P=l(a),F=i(a,"P",{"data-svelte-h":!0}),o(F)!=="svelte-n6wfk7"&&(F.innerHTML=fa),K=l(a),h(g.$$.fragment,a),O=l(a),h(D.$$.fragment,a),aa=l(a),Q=i(a,"P",{"data-svelte-h":!0}),o(Q)!=="svelte-9b4dmt"&&(Q.innerHTML=ya),sa=l(a),h(G.$$.fragment,a),ta=l(a),q=i(a,"P",{"data-svelte-h":!0}),o(q)!=="svelte-1l78fpm"&&(q.innerHTML=ga),ea=l(a),h(Z.$$.fragment,a),na=l(a),v=i(a,"P",{}),Ja(v).forEach(t),this.h()},h(){Ta(p,"name","hf:doc:metadata"),Ta(p,"content",Da)},m(a,s){ba(document.head,p),e(a,M,s),e(a,m,s),e(a,Y,s),u(J,a,s),e(a,S,s),e(a,T,s),e(a,z,s),e(a,j,s),e(a,X,s),u(w,a,s),e(a,x,s),e(a,U,s),e(a,B,s),e(a,k,s),e(a,E,s),u(I,a,s),e(a,H,s),e(a,$,s),e(a,N,s),e(a,b,s),e(a,L,s),u(_,a,s),e(a,V,s),u(C,a,s),e(a,A,s),e(a,R,s),e(a,P,s),e(a,F,s),e(a,K,s),u(g,a,s),e(a,O,s),u(D,a,s),e(a,aa,s),e(a,Q,s),e(a,sa,s),u(G,a,s),e(a,ta,s),e(a,q,s),e(a,ea,s),u(Z,a,s),e(a,na,s),e(a,v,s),la=!0},p(a,[s]){const Ma={};s&2&&(Ma.$$scope={dirty:s,ctx:a}),g.$set(Ma)},i(a){la||(d(J.$$.fragment,a),d(w.$$.fragment,a),d(I.$$.fragment,a),d(_.$$.fragment,a),d(C.$$.fragment,a),d(g.$$.fragment,a),d(D.$$.fragment,a),d(G.$$.fragment,a),d(Z.$$.fragment,a),la=!0)},o(a){f(J.$$.fragment,a),f(w.$$.fragment,a),f(I.$$.fragment,a),f(_.$$.fragment,a),f(C.$$.fragment,a),f(g.$$.fragment,a),f(D.$$.fragment,a),f(G.$$.fragment,a),f(Z.$$.fragment,a),la=!1},d(a){a&&(t(M),t(m),t(Y),t(S),t(T),t(z),t(j),t(X),t(x),t(U),t(B),t(k),t(E),t(H),t($),t(N),t(b),t(L),t(V),t(A),t(R),t(P),t(F),t(K),t(O),t(aa),t(Q),t(sa),t(ta),t(q),t(ea),t(na),t(v)),t(p),y(J,a),y(w,a),y(I,a),y(_,a),y(C,a),y(g,a),y(D,a),y(G,a),y(Z,a)}}}const Da='{"title":"Use with Spark","local":"use-with-spark","sections":[{"title":"Load from Spark","local":"load-from-spark","sections":[{"title":"Caching","local":"caching","sections":[],"depth":3},{"title":"Feature types","local":"feature-types","sections":[],"depth":3}],"depth":2}],"depth":1}';function Qa(W){return wa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Wa extends ka{constructor(p){super(),Ia(this,p,Qa,Fa,ja,{})}}export{Wa as component};
