import{s as Ts,n as rs,o as hs}from"../chunks/scheduler.d6170356.js";import{S as js,i as os,g as M,s as t,r as U,A as ws,h as p,f as a,c as e,j as Js,u as i,x as y,k as vl,y as ms,a as n,v as c,d as J,t as T,w as r}from"../chunks/index.fcd4cc08.js";import{C as h}from"../chunks/CodeBlock.7b16bdef.js";import{H as al,E as us}from"../chunks/EditOnGithub.da2b595c.js";function ds(_l){let j,nl,ll,tl,m,el,u,Hl="Spark enables real-time, large-scale data processing in a distributed environment.",Ml,d,ql="In particular you can use <code>huggingface_hub</code> to access Hugging Face datasets repositories in PySpark",pl,b,yl,I,xl="To be able to read and write to Hugging Face URLs (e.g. <code>hf://datasets/username/dataset/data.parquet</code>), you need to install the <code>huggingface_hub</code> library:",Ul,C,il,B,Dl="You also need to install <code>pyarrow</code> to read/write Parquet / JSON / CSV / etc. files using the filesystem API provided by <code>huggingFace_hub</code>:",cl,g,Jl,G,Tl,f,Ol="You need to authenticate to Hugging Face to read private/gated dataset repositories or to write to your dataset repositories.",rl,k,Ll="You can use the CLI for example:",hl,Z,jl,N,$l="It’s also possible to provide your Hugging Face token with the <code>HF_TOKEN</code> environment variable or passing the <code>storage_options</code> parameter to helper functions below:",ol,R,wl,S,Kl='For more details about authentication, check out <a href="https://huggingface.co/docs/huggingface_hub/quick-start#authentication" rel="nofollow">this guide</a>.',ml,Q,ul,A,Pl="PySpark doesn’t have an official support for Hugging Face paths, so we provide a helper function to read datasets in a distributed manner.",dl,F,ls="For example you can read Parquet files from Hugging Face in an optimized way using PyArrow by defining this <code>read_parquet</code> helper function:",bl,Y,Il,X,ss=`Here is how we can use this on the <a href="https://huggingface.co/datasets/BAAI/Infinity-Instruct" rel="nofollow">BAAI/Infinity-Instruct</a> dataset.
It is a gated repository, users have to accept the terms of use before accessing it.`,Cl,o,as='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-7M-min.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-7M-dark-min.png"/>',Bl,W,ns="We use the <code>read_parquet</code> function to read data from the dataset, compute the number of dialogue per language and filter the dataset.",gl,V,ts="After logging-in to access the gated repository, we can run:",Gl,E,fl,z,es=`To compute the number of dialogues per language we run this code.
The <code>columns</code> argument is useful to only load the data we need, since PySpark doesn’t enable predicate push-down in this case.
There is also a <code>filters</code> argument to only load data with values within a certain range.`,kl,v,Zl,_,Ms="To filter the dataset and only keep dialogues in Chinese:",Nl,H,Rl,q,Sl,x,ps="We also provide a helper function to write datasets in a distributed manner to a Hugging Face repository.",Ql,D,ys=`You can write a PySpark Dataframe to Hugging Face using this <code>write_parquet</code> helper functions based on the <code>huggingface_hub</code> API.
In particular it uses the <code>preupload_lfs_files</code> utility to upload Parquet files in parallel in a distributed manner, and only commit the files once they’re all uploaded:`,Al,O,Fl,L,Us='Here is how we can use this function to write the filtered version of the <a href="https://huggingface.co/datasets/BAAI/Infinity-Instruct" rel="nofollow">BAAI/Infinity-Instruct</a> dataset back to Hugging Face.',Yl,$,is=`First you need to <a href="https://huggingface.co/new-dataset" rel="nofollow">create a dataset repository</a>, e.g. <code>username/Infinity-Instruct-Chinese-Only</code> (you can set it to private if you want).
Then, make sure you are authenticated and you can run:`,Xl,K,Wl,w,cs='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-chinese-only-min.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-chinese-only-dark-min.png"/>',Vl,P,El,sl,zl;return m=new al({props:{title:"Spark",local:"spark",headingTag:"h1"}}),b=new al({props:{title:"Installation",local:"installation",headingTag:"h2"}}),C=new h({props:{code:"cGlwJTIwaW5zdGFsbCUyMGh1Z2dpbmdmYWNlX2h1Yg==",highlighted:'pip <span class="hljs-keyword">install</span> huggingface_hub',wrap:!1}}),g=new h({props:{code:"cGlwJTIwaW5zdGFsbCUyMHB5YXJyb3c=",highlighted:'pip <span class="hljs-keyword">install</span> pyarrow',wrap:!1}}),G=new al({props:{title:"Authentication",local:"authentication",headingTag:"h2"}}),Z=new h({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:'huggingface-<span class="hljs-keyword">cli</span> login',wrap:!1}}),R=new h({props:{code:"c3RvcmFnZV9vcHRpb25zJTIwJTNEJTIwJTdCJTIydG9rZW4lMjIlM0ElMjAlMjJoZl94eHglMjIlN0Q=",highlighted:'storage_options = {<span class="hljs-string">&quot;token&quot;</span>: <span class="hljs-string">&quot;hf_xxx&quot;</span>}',wrap:!1}}),Q=new al({props:{title:"Read",local:"read",headingTag:"h2"}}),Y=new h({props:{code:" ",highlighted:`<span class="hljs-keyword">from</span> functools <span class="hljs-keyword">import</span> partial
<span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> Iterator, <span class="hljs-type">Optional</span>, <span class="hljs-type">Union</span>

<span class="hljs-keyword">import</span> pyarrow <span class="hljs-keyword">as</span> pa
<span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq
<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfFileSystem
<span class="hljs-keyword">from</span> pyspark.sql.dataframe <span class="hljs-keyword">import</span> DataFrame
<span class="hljs-keyword">from</span> pyspark.sql.pandas.types <span class="hljs-keyword">import</span> from_arrow_schema


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_read</span>(<span class="hljs-params">iterator: Iterator[pa.RecordBatch], **kwargs</span>) -&gt; Iterator[pa.RecordBatch]:
    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> iterator:
        <span class="hljs-keyword">for</span> path <span class="hljs-keyword">in</span> batch[<span class="hljs-number">0</span>].to_pylist():
            <span class="hljs-keyword">yield</span> <span class="hljs-keyword">from</span> pq.read_table(path, **kwargs).to_batches()


<span class="hljs-keyword">def</span> <span class="hljs-title function_">read_parquet</span>(<span class="hljs-params">
    path: <span class="hljs-built_in">str</span>,
    columns: <span class="hljs-type">Optional</span>[<span class="hljs-built_in">list</span>[<span class="hljs-built_in">str</span>]] = <span class="hljs-literal">None</span>,
    filters: <span class="hljs-type">Optional</span>[<span class="hljs-type">Union</span>[<span class="hljs-built_in">list</span>[<span class="hljs-built_in">tuple</span>], <span class="hljs-built_in">list</span>[<span class="hljs-built_in">list</span>[<span class="hljs-built_in">tuple</span>]]]] = <span class="hljs-literal">None</span>,
    **kwargs,
</span>) -&gt; DataFrame:
    <span class="hljs-string">&quot;&quot;&quot;
    Loads Parquet files from Hugging Face using PyArrow, returning a PySPark \`DataFrame\`.

    It reads Parquet files in a distributed manner.

    Access private or gated repositories using \`huggingface-cli login\` or passing a token
    using the \`storage_options\` argument: \`storage_options={&quot;token&quot;: &quot;hf_xxx&quot;}\`

    Parameters
    ----------
    path : str
        Path to the file. Prefix with a protocol like \`hf://\` to read from Hugging Face.
        You can read from multiple files if you pass a globstring.
    columns : list, default None
        If not None, only these columns will be read from the file.
    filters : List[Tuple] or List[List[Tuple]], default None
        To filter out data.
        Filter syntax: [[(column, op, val), ...],...]
        where op is [==, =, &gt;, &gt;=, &lt;, &lt;=, !=, in, not in]
        The innermost tuples are transposed into a set of filters applied
        through an \`AND\` operation.
        The outer list combines these sets of filters through an \`OR\`
        operation.
        A single list of tuples can also be used, meaning that no \`OR\`
        operation between set of filters is to be conducted.

    **kwargs
        Any additional kwargs are passed to pyarrow.parquet.read_table.

    Returns
    -------
    DataFrame
        DataFrame based on parquet file.

    Examples
    --------
    &gt;&gt;&gt; path = &quot;hf://datasets/username/dataset/data.parquet&quot;
    &gt;&gt;&gt; pd.DataFrame({&quot;foo&quot;: range(5), &quot;bar&quot;: range(5, 10)}).to_parquet(path)
    &gt;&gt;&gt; read_parquet(path).show()
    +---+---+
    |foo|bar|
    +---+---+
    |  0|  5|
    |  1|  6|
    |  2|  7|
    |  3|  8|
    |  4|  9|
    +---+---+
    &gt;&gt;&gt; read_parquet(path, columns=[&quot;bar&quot;]).show()
    +---+
    |bar|
    +---+
    |  5|
    |  6|
    |  7|
    |  8|
    |  9|
    +---+
    &gt;&gt;&gt; sel = [(&quot;foo&quot;, &quot;&gt;&quot;, 2)]
    &gt;&gt;&gt; read_parquet(path, filters=sel).show()
    +---+---+
    |foo|bar|
    +---+---+
    |  3|  8|
    |  4|  9|
    +---+---+
    &quot;&quot;&quot;</span>
    filesystem: HfFileSystem = kwargs.pop(<span class="hljs-string">&quot;filesystem&quot;</span>) <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;filesystem&quot;</span> <span class="hljs-keyword">in</span> kwargs <span class="hljs-keyword">else</span> HfFileSystem(**kwargs.pop(<span class="hljs-string">&quot;storage_options&quot;</span>, {}))
    paths = filesystem.glob(path)
    <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> paths:
        <span class="hljs-keyword">raise</span> FileNotFoundError(<span class="hljs-string">f&quot;Counldn&#x27;t find any file at <span class="hljs-subst">{path}</span>&quot;</span>)
    rdd = spark.sparkContext.parallelize([{<span class="hljs-string">&quot;path&quot;</span>: path} <span class="hljs-keyword">for</span> path <span class="hljs-keyword">in</span> paths], <span class="hljs-built_in">len</span>(paths))
    df = spark.createDataFrame(rdd)
    arrow_schema = pq.read_schema(filesystem.<span class="hljs-built_in">open</span>(paths[<span class="hljs-number">0</span>]))
    schema = pa.schema([field <span class="hljs-keyword">for</span> field <span class="hljs-keyword">in</span> arrow_schema <span class="hljs-keyword">if</span> (columns <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> field.name <span class="hljs-keyword">in</span> columns)], metadata=arrow_schema.metadata)
    <span class="hljs-keyword">return</span> df.mapInArrow(
        partial(_read, columns=columns, filters=filters, filesystem=filesystem, schema=schema, **kwargs),
        from_arrow_schema(schema),
    )`,wrap:!1}}),E=new h({props:{code:"ZnJvbSUyMHB5c3Bhcmsuc3FsJTIwaW1wb3J0JTIwU3BhcmtTZXNzaW9uJTBBc3BhcmslMjAlM0QlMjBTcGFya1Nlc3Npb24uYnVpbGRlci5hcHBOYW1lKCUyMmRlbW8lMjIpLmdldE9yQ3JlYXRlKCklMEFkZiUyMCUzRCUyMHJlYWRfcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGQkFBSSUyRkluZmluaXR5LUluc3RydWN0JTJGN00lMkYqLnBhcnF1ZXQlMjIpJTBBZGYuc2hvdygp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> pyspark.sql <span class="hljs-keyword">import</span> SparkSession
<span class="hljs-meta">&gt;&gt;&gt; </span>spark = SparkSession.builder.appName(<span class="hljs-string">&quot;demo&quot;</span>).getOrCreate()
<span class="hljs-meta">&gt;&gt;&gt; </span>df = read_parquet(<span class="hljs-string">&quot;hf://datasets/BAAI/Infinity-Instruct/7M/*.parquet&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>df.show()
+---+----------------------------+-----+----------+--------------------+        
| <span class="hljs-built_in">id</span>|               conversations|label|langdetect|              source|
+---+----------------------------+-----+----------+--------------------+
|  <span class="hljs-number">0</span>|        [{human, <span class="hljs-keyword">def</span> <span class="hljs-title function_">exti</span>...|     |        en|      code_exercises|
|  <span class="hljs-number">1</span>|        [{human, See the ...|     |        en|                flan|
|  <span class="hljs-number">2</span>|        [{human, This <span class="hljs-keyword">is</span> ...|     |        en|                flan|
|  <span class="hljs-number">3</span>|        [{human, If you d...|     |        en|                flan|
|  <span class="hljs-number">4</span>|        [{human, In a Uni...|     |        en|                flan|
|  <span class="hljs-number">5</span>|        [{human, Read the...|     |        en|                flan|
|  <span class="hljs-number">6</span>|        [{human, You are ...|     |        en|          code_bagel|
|  <span class="hljs-number">7</span>|        [{human, I want y...|     |        en|          Subjective|
|  <span class="hljs-number">8</span>|        [{human, Given th...|     |        en|                flan|
|  <span class="hljs-number">9</span>|[{human, 因果联系原则是法...|     |     zh-cn|          Subjective|
| <span class="hljs-number">10</span>|        [{human, Provide ...|     |        en|self-oss-instruct...|
| <span class="hljs-number">11</span>|        [{human, The univ...|     |        en|                flan|
| <span class="hljs-number">12</span>|        [{human, Q: I am ...|     |        en|                flan|
| <span class="hljs-number">13</span>|        [{human, What <span class="hljs-keyword">is</span> ...|     |        en|      OpenHermes-<span class="hljs-number">2.5</span>|
| <span class="hljs-number">14</span>|        [{human, In react...|     |        en|                flan|
| <span class="hljs-number">15</span>|        [{human, Write Py...|     |        en|      code_exercises|
| <span class="hljs-number">16</span>|        [{human, Find the...|     |        en|            MetaMath|
| <span class="hljs-number">17</span>|        [{human, Three of...|     |        en|            MetaMath|
| <span class="hljs-number">18</span>|        [{human, Chandra ...|     |        en|            MetaMath|
| <span class="hljs-number">19</span>|[{human, 用经济学知识分析...|     |     zh-cn|          Subjective|
+---+----------------------------+-----+----------+--------------------+`,wrap:!1}}),v=new h({props:{code:"ZGZfbGFuZ2RldGVjdF9vbmx5JTIwJTNEJTIwcmVhZF9wYXJxdWV0KCUyMmhmJTNBJTJGJTJGZGF0YXNldHMlMkZCQUFJJTJGSW5maW5pdHktSW5zdHJ1Y3QlMkY3TSUyRioucGFycXVldCUyMiUyQyUyMGNvbHVtbnMlM0QlNUIlMjJsYW5nZGV0ZWN0JTIyJTVEKSUwQWRmX2xhbmdkZXRlY3Rfb25seS5ncm91cEJ5KCUyMmxhbmdkZXRlY3QlMjIpLmNvdW50KCkuc2hvdygp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>df_langdetect_only = read_parquet(<span class="hljs-string">&quot;hf://datasets/BAAI/Infinity-Instruct/7M/*.parquet&quot;</span>, columns=[<span class="hljs-string">&quot;langdetect&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>df_langdetect_only.groupBy(<span class="hljs-string">&quot;langdetect&quot;</span>).count().show()
+----------+-------+                                                            
|langdetect|  count|
+----------+-------+
|        en|<span class="hljs-number">6697793</span>|
|     zh-cn| <span class="hljs-number">751313</span>|
+----------+-------+`,wrap:!1}}),H=new h({props:{code:"Y3JpdGVyaWElMjAlM0QlMjAlNUIoJTIybGFuZ2RldGVjdCUyMiUyQyUyMCUyMiUzRCUyMiUyQyUyMCUyMnpoLWNuJTIyKSU1RCUwQWRmX2NoaW5lc2Vfb25seSUyMCUzRCUyMHJlYWRfcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGQkFBSSUyRkluZmluaXR5LUluc3RydWN0JTJGN00lMkYqLnBhcnF1ZXQlMjIlMkMlMjBmaWx0ZXJzJTNEY3JpdGVyaWEpJTBBZGZfY2hpbmVzZV9vbmx5",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>criteria = [(<span class="hljs-string">&quot;langdetect&quot;</span>, <span class="hljs-string">&quot;=&quot;</span>, <span class="hljs-string">&quot;zh-cn&quot;</span>)]
<span class="hljs-meta">&gt;&gt;&gt; </span>df_chinese_only = read_parquet(<span class="hljs-string">&quot;hf://datasets/BAAI/Infinity-Instruct/7M/*.parquet&quot;</span>, filters=criteria)
<span class="hljs-meta">&gt;&gt;&gt; </span>df_chinese_only
+---+----------------------------+-----+----------+----------+                  
| <span class="hljs-built_in">id</span>|               conversations|label|langdetect|    source|
+---+----------------------------+-----+----------+----------+
|  <span class="hljs-number">9</span>|[{human, 因果联系原则是法...|     |     zh-cn|Subjective|
| <span class="hljs-number">19</span>|[{human, 用经济学知识分析...|     |     zh-cn|Subjective|
| <span class="hljs-number">38</span>| [{human, 某个考试共有A、...|     |     zh-cn|Subjective|
| <span class="hljs-number">39</span>|[{human, 撰写一篇关于斐波...|     |     zh-cn|Subjective|
| <span class="hljs-number">57</span>|[{human, 总结世界历史上的...|     |     zh-cn|Subjective|
| <span class="hljs-number">61</span>|[{human, 生成一则广告词。...|     |     zh-cn|Subjective|
| <span class="hljs-number">66</span>|[{human, 描述一个有效的团...|     |     zh-cn|Subjective|
| <span class="hljs-number">94</span>|[{human, 如果比利和蒂芙尼...|     |     zh-cn|Subjective|
|<span class="hljs-number">102</span>|[{human, 生成一句英文名言...|     |     zh-cn|Subjective|
|<span class="hljs-number">106</span>|[{human, 写一封感谢信，感...|     |     zh-cn|Subjective|
|<span class="hljs-number">118</span>| [{human, 生成一个故事。}...|     |     zh-cn|Subjective|
|<span class="hljs-number">174</span>|[{human, 高胆固醇水平的后...|     |     zh-cn|Subjective|
|<span class="hljs-number">180</span>|[{human, 基于以下角色信息...|     |     zh-cn|Subjective|
|<span class="hljs-number">192</span>|[{human, 请写一篇文章，概...|     |     zh-cn|Subjective|
|<span class="hljs-number">221</span>|[{human, 以诗歌形式表达对...|     |     zh-cn|Subjective|
|<span class="hljs-number">228</span>|[{human, 根据给定的指令，...|     |     zh-cn|Subjective|
|<span class="hljs-number">236</span>|[{human, 打开一个新的生成...|     |     zh-cn|Subjective|
|<span class="hljs-number">260</span>|[{human, 生成一个有关未来...|     |     zh-cn|Subjective|
|<span class="hljs-number">268</span>|[{human, 如果有一定数量的...|     |     zh-cn|Subjective|
|<span class="hljs-number">273</span>| [{human, 题目：小明有<span class="hljs-number">5</span>个...|     |     zh-cn|Subjective|
+---+----------------------------+-----+----------+----------+`,wrap:!1}}),q=new al({props:{title:"Write",local:"write",headingTag:"h2"}}),O=new h({props:{code:" ",highlighted:`<span class="hljs-keyword">import</span> math
<span class="hljs-keyword">import</span> pickle
<span class="hljs-keyword">import</span> tempfile
<span class="hljs-keyword">from</span> functools <span class="hljs-keyword">import</span> partial
<span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> Iterator, <span class="hljs-type">Optional</span>

<span class="hljs-keyword">import</span> pyarrow <span class="hljs-keyword">as</span> pa
<span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq
<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> CommitOperationAdd, HfFileSystem
<span class="hljs-keyword">from</span> pyspark.sql.dataframe <span class="hljs-keyword">import</span> DataFrame
<span class="hljs-keyword">from</span> pyspark.sql.pandas.types <span class="hljs-keyword">import</span> from_arrow_schema, to_arrow_schema


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_preupload</span>(<span class="hljs-params">iterator: Iterator[pa.RecordBatch], path: <span class="hljs-built_in">str</span>, schema: pa.Schema, filesystem: HfFileSystem, row_group_size: <span class="hljs-type">Optional</span>[<span class="hljs-built_in">int</span>] = <span class="hljs-literal">None</span>, **kwargs</span>) -&gt; Iterator[pa.RecordBatch]:
    resolved_path = filesystem.resolve_path(path)
    <span class="hljs-keyword">with</span> tempfile.NamedTemporaryFile(suffix=<span class="hljs-string">&quot;.parquet&quot;</span>) <span class="hljs-keyword">as</span> temp_file:
        <span class="hljs-keyword">with</span> pq.ParquetWriter(temp_file.name, schema=schema, **kwargs) <span class="hljs-keyword">as</span> writer:
            <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> iterator:
                writer.write_batch(batch, row_group_size=row_group_size)
        addition = CommitOperationAdd(path_in_repo=temp_file.name, path_or_fileobj=temp_file.name)
        filesystem._api.preupload_lfs_files(repo_id=resolved_path.repo_id, additions=[addition], repo_type=resolved_path.repo_type, revision=resolved_path.revision)
    <span class="hljs-keyword">yield</span> pa.record_batch({<span class="hljs-string">&quot;addition&quot;</span>: [pickle.dumps(addition)]}, schema=pa.schema({<span class="hljs-string">&quot;addition&quot;</span>: pa.binary()}))


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_commit</span>(<span class="hljs-params">iterator: Iterator[pa.RecordBatch], path: <span class="hljs-built_in">str</span>, filesystem: HfFileSystem, max_operations_per_commit=<span class="hljs-number">50</span></span>) -&gt; Iterator[pa.RecordBatch]:
    resolved_path = filesystem.resolve_path(path)
    additions: <span class="hljs-built_in">list</span>[CommitOperationAdd] = [pickle.loads(addition) <span class="hljs-keyword">for</span> addition <span class="hljs-keyword">in</span> pa.Table.from_batches(iterator, schema=pa.schema({<span class="hljs-string">&quot;addition&quot;</span>: pa.binary()}))[<span class="hljs-number">0</span>].to_pylist()]
    num_commits = math.ceil(<span class="hljs-built_in">len</span>(additions) / max_operations_per_commit)
    <span class="hljs-keyword">for</span> shard_idx, addition <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(additions):
        addition.path_in_repo = resolved_path.path_in_repo.replace(<span class="hljs-string">&quot;{shard_idx:05d}&quot;</span>, <span class="hljs-string">f&quot;<span class="hljs-subst">{shard_idx:05d}</span>&quot;</span>)
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, num_commits):
        operations = additions[i * max_operations_per_commit : (i + <span class="hljs-number">1</span>) * max_operations_per_commit]
        commit_message = <span class="hljs-string">&quot;Upload using PySpark&quot;</span> + (<span class="hljs-string">f&quot; (part <span class="hljs-subst">{i:05d}</span>-of-<span class="hljs-subst">{num_commits:05d}</span>)&quot;</span> <span class="hljs-keyword">if</span> num_commits &gt; <span class="hljs-number">1</span> <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;&quot;</span>)
        filesystem._api.create_commit(repo_id=resolved_path.repo_id, repo_type=resolved_path.repo_type, revision=resolved_path.revision, operations=operations, commit_message=commit_message)
        <span class="hljs-keyword">yield</span> pa.record_batch({<span class="hljs-string">&quot;path&quot;</span>: [addition.path_in_repo <span class="hljs-keyword">for</span> addition <span class="hljs-keyword">in</span> operations]}, schema=pa.schema({<span class="hljs-string">&quot;path&quot;</span>: pa.string()}))


<span class="hljs-keyword">def</span> <span class="hljs-title function_">write_parquet</span>(<span class="hljs-params">df: DataFrame, path: <span class="hljs-built_in">str</span>, **kwargs</span>) -&gt; <span class="hljs-literal">None</span>:
    <span class="hljs-string">&quot;&quot;&quot;
    Write Parquet files to Hugging Face using PyArrow.

    It uploads Parquet files in a distributed manner in two steps:

    1. Preupload the Parquet files in parallel in a distributed banner
    2. Commit the preuploaded files

    Authenticate using \`huggingface-cli login\` or passing a token
    using the \`storage_options\` argument: \`storage_options={&quot;token&quot;: &quot;hf_xxx&quot;}\`

    Parameters
    ----------
    path : str
        Path of the file or directory. Prefix with a protocol like \`hf://\` to read from Hugging Face.
        It writes Parquet files in the form &quot;part-xxxxx.parquet&quot;, or to a single file if \`path ends with &quot;.parquet&quot;.

    **kwargs
        Any additional kwargs are passed to pyarrow.parquet.ParquetWriter.

    Returns
    -------
    DataFrame
        DataFrame based on parquet file.

    Examples
    --------
    &gt;&gt;&gt; spark.createDataFrame(pd.DataFrame({&quot;foo&quot;: range(5), &quot;bar&quot;: range(5, 10)}))
    &gt;&gt;&gt; # Save to one file
    &gt;&gt;&gt; write_parquet(df, &quot;hf://datasets/username/dataset/data.parquet&quot;)
    &gt;&gt;&gt; # OR save to a directory (possibly in many files)
    &gt;&gt;&gt; write_parquet(df, &quot;hf://datasets/username/dataset&quot;)
    &quot;&quot;&quot;</span>
    filesystem: HfFileSystem = kwargs.pop(<span class="hljs-string">&quot;filesystem&quot;</span>, HfFileSystem(**kwargs.pop(<span class="hljs-string">&quot;storage_options&quot;</span>, {})))
    <span class="hljs-keyword">if</span> path.endswith(<span class="hljs-string">&quot;.parquet&quot;</span>) <span class="hljs-keyword">or</span> path.endswith(<span class="hljs-string">&quot;.pq&quot;</span>):
        df = df.coalesce(<span class="hljs-number">1</span>)
    <span class="hljs-keyword">else</span>:
        path += <span class="hljs-string">&quot;/part-{shard_idx:05d}.parquet&quot;</span>
    df.mapInArrow(
        partial(_preupload, path=path, schema=to_arrow_schema(df.schema), filesystem=filesystem, **kwargs),
        from_arrow_schema(pa.schema({<span class="hljs-string">&quot;addition&quot;</span>: pa.binary()})),
    ).coalesce(<span class="hljs-number">1</span>).mapInArrow(
        partial(_commit, path=path, filesystem=filesystem),
        from_arrow_schema(pa.schema({<span class="hljs-string">&quot;path&quot;</span>: pa.string()})),
    ).collect()`,wrap:!1}}),K=new h({props:{code:"d3JpdGVfcGFycXVldChkZl9jaGluZXNlX29ubHklMkMlMjAlMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZJbmZpbml0eS1JbnN0cnVjdC1DaGluZXNlLU9ubHklMjIpJTBBdG1waDlqd3U5cHkucGFycXVldCUzQSUyMDEwMCUyNSU3QyVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCU3QyUyMDUwLjVNJTJGNTAuNU0lMjAlNUIwMCUzQTAzJTNDMDAlM0EwMCUyQyUyMDE0LjZNQiUyRnMlNUQlMEF0bXAwb3F0OTluYy5wYXJxdWV0JTNBJTIwMTAwJTI1JTdDJUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JTdDJTIwNTAuOE0lMkY1MC44TSUyMCU1QjAwJTNBMDIlM0MwMCUzQTAwJTJDJTIwMTcuOU1CJTJGcyU1RCUwQXRtcGduaXprd3FwLnBhcnF1ZXQlM0ElMjAxMDAlMjUlN0MlRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglN0MlMjA1MC41TSUyRjUwLjVNJTIwJTVCMDAlM0EwMiUzQzAwJTNBMDAlMkMlMjAxOS42TUIlMkZzJTVEJTBBdG1wYW5tMDRrNG4ucGFycXVldCUzQSUyMDEwMCUyNSU3QyVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCU3QyUyMDUxLjRNJTJGNTEuNE0lMjAlNUIwMCUzQTAyJTNDMDAlM0EwMCUyQyUyMDIyLjlNQiUyRnMlNUQlMEF0bXAxNHV5OW9xYi5wYXJxdWV0JTNBJTIwMTAwJTI1JTdDJUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JTdDJTIwNTAuNE0lMkY1MC40TSUyMCU1QjAwJTNBMDIlM0MwMCUzQTAwJTJDJTIwMjMuME1CJTJGcyU1RCUwQXRtcGNwOHRfcWRsLnBhcnF1ZXQlM0ElMjAxMDAlMjUlN0MlRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglN0MlMjA1MC40TSUyRjUwLjRNJTIwJTVCMDAlM0EwMiUzQzAwJTNBMDAlMkMlMjAyMy41TUIlMkZzJTVEJTBBdG1wanVpNW1uczgucGFycXVldCUzQSUyMDEwMCUyNSU3QyVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCU3QyUyMDUwLjNNJTJGNTAuM00lMjAlNUIwMCUzQTAyJTNDMDAlM0EwMCUyQyUyMDI0LjFNQiUyRnMlNUQlMEF0bXB5ZHFoNm9kMS5wYXJxdWV0JTNBJTIwMTAwJTI1JTdDJUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JTdDJTIwNTAuOU0lMkY1MC45TSUyMCU1QjAwJTNBMDIlM0MwMCUzQTAwJTJDJTIwMjMuOE1CJTJGcyU1RCUwQXRtcDUyZjJ0OHR1LnBhcnF1ZXQlM0ElMjAxMDAlMjUlN0MlRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglN0MlMjA1MC41TSUyRjUwLjVNJTIwJTVCMDAlM0EwMiUzQzAwJTNBMDAlMkMlMjAyMy43TUIlMkZzJTVEJTBBdG1wZzdlZ3YzeWUucGFycXVldCUzQSUyMDEwMCUyNSU3QyVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCVFMiU5NiU4OCU3QyUyMDUwLjFNJTJGNTAuMU0lMjAlNUIwMCUzQTA2JTNDMDAlM0EwMCUyQyUyMDcuNjhNQiUyRnMlNUQlMEF0bXAyczBmcTJobS5wYXJxdWV0JTNBJTIwMTAwJTI1JTdDJUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JUUyJTk2JTg4JTdDJTIwNTAuOE0lMkY1MC44TSUyMCU1QjAwJTNBMDIlM0MwMCUzQTAwJTJDJTIwMTguMU1CJTJGcyU1RCUwQXRtcG1qOTdhYjMwLnBhcnF1ZXQlM0ElMjAxMDAlMjUlN0MlRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglRTIlOTYlODglN0MlMjA3MS4zTSUyRjcxLjNNJTIwJTVCMDAlM0EwMiUzQzAwJTNBMDAlMkMlMjAyMy45TUIlMkZzJTVE",highlighted:`write_parquet(df_chinese_only, <span class="hljs-string">&quot;hf://datasets/username/Infinity-Instruct-Chinese-Only&quot;</span>)
tmph9jwu9py.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.5</span>M/<span class="hljs-number">50.5</span>M [<span class="hljs-number">00</span>:03&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">14.6</span>MB/s]
tmp0oqt99nc.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.8</span>M/<span class="hljs-number">50.8</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">17.9</span>MB/s]
tmpgnizkwqp.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.5</span>M/<span class="hljs-number">50.5</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">19.6</span>MB/s]
tmpanm04k4n.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">51.4</span>M/<span class="hljs-number">51.4</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">22.9</span>MB/s]
tmp14uy9oqb.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.4</span>M/<span class="hljs-number">50.4</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">23.0</span>MB/s]
tmpcp8t_qdl.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.4</span>M/<span class="hljs-number">50.4</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">23.5</span>MB/s]
tmpjui5mns8.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.3</span>M/<span class="hljs-number">50.3</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">24.1</span>MB/s]
tmpydqh6od1.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.9</span>M/<span class="hljs-number">50.9</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">23.8</span>MB/s]
tmp52f2t8tu.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.5</span>M/<span class="hljs-number">50.5</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">23.7</span>MB/s]
tmpg7egv3ye.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.1</span>M/<span class="hljs-number">50.1</span>M [<span class="hljs-number">00</span>:06&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">7.68</span>MB/s]
tmp2s0fq2hm.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">50.8</span>M/<span class="hljs-number">50.8</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">18.1</span>MB/s]
tmpmj97ab30.parquet: <span class="hljs-number">100</span>%|██████████| <span class="hljs-number">71.3</span>M/<span class="hljs-number">71.3</span>M [<span class="hljs-number">00</span>:02&lt;<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">23.9</span>MB/s]`,wrap:!1}}),P=new us({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-spark.md"}}),{c(){j=M("meta"),nl=t(),ll=M("p"),tl=t(),U(m.$$.fragment),el=t(),u=M("p"),u.textContent=Hl,Ml=t(),d=M("p"),d.innerHTML=ql,pl=t(),U(b.$$.fragment),yl=t(),I=M("p"),I.innerHTML=xl,Ul=t(),U(C.$$.fragment),il=t(),B=M("p"),B.innerHTML=Dl,cl=t(),U(g.$$.fragment),Jl=t(),U(G.$$.fragment),Tl=t(),f=M("p"),f.textContent=Ol,rl=t(),k=M("p"),k.textContent=Ll,hl=t(),U(Z.$$.fragment),jl=t(),N=M("p"),N.innerHTML=$l,ol=t(),U(R.$$.fragment),wl=t(),S=M("p"),S.innerHTML=Kl,ml=t(),U(Q.$$.fragment),ul=t(),A=M("p"),A.textContent=Pl,dl=t(),F=M("p"),F.innerHTML=ls,bl=t(),U(Y.$$.fragment),Il=t(),X=M("p"),X.innerHTML=ss,Cl=t(),o=M("div"),o.innerHTML=as,Bl=t(),W=M("p"),W.innerHTML=ns,gl=t(),V=M("p"),V.textContent=ts,Gl=t(),U(E.$$.fragment),fl=t(),z=M("p"),z.innerHTML=es,kl=t(),U(v.$$.fragment),Zl=t(),_=M("p"),_.textContent=Ms,Nl=t(),U(H.$$.fragment),Rl=t(),U(q.$$.fragment),Sl=t(),x=M("p"),x.textContent=ps,Ql=t(),D=M("p"),D.innerHTML=ys,Al=t(),U(O.$$.fragment),Fl=t(),L=M("p"),L.innerHTML=Us,Yl=t(),$=M("p"),$.innerHTML=is,Xl=t(),U(K.$$.fragment),Wl=t(),w=M("div"),w.innerHTML=cs,Vl=t(),U(P.$$.fragment),El=t(),sl=M("p"),this.h()},l(l){const s=ws("svelte-u9bgzb",document.head);j=p(s,"META",{name:!0,content:!0}),s.forEach(a),nl=e(l),ll=p(l,"P",{}),Js(ll).forEach(a),tl=e(l),i(m.$$.fragment,l),el=e(l),u=p(l,"P",{"data-svelte-h":!0}),y(u)!=="svelte-s0r53o"&&(u.textContent=Hl),Ml=e(l),d=p(l,"P",{"data-svelte-h":!0}),y(d)!=="svelte-12z8vci"&&(d.innerHTML=ql),pl=e(l),i(b.$$.fragment,l),yl=e(l),I=p(l,"P",{"data-svelte-h":!0}),y(I)!=="svelte-13h0lcm"&&(I.innerHTML=xl),Ul=e(l),i(C.$$.fragment,l),il=e(l),B=p(l,"P",{"data-svelte-h":!0}),y(B)!=="svelte-ml7hwl"&&(B.innerHTML=Dl),cl=e(l),i(g.$$.fragment,l),Jl=e(l),i(G.$$.fragment,l),Tl=e(l),f=p(l,"P",{"data-svelte-h":!0}),y(f)!=="svelte-aqw7kt"&&(f.textContent=Ol),rl=e(l),k=p(l,"P",{"data-svelte-h":!0}),y(k)!=="svelte-pdivv8"&&(k.textContent=Ll),hl=e(l),i(Z.$$.fragment,l),jl=e(l),N=p(l,"P",{"data-svelte-h":!0}),y(N)!=="svelte-1pzllua"&&(N.innerHTML=$l),ol=e(l),i(R.$$.fragment,l),wl=e(l),S=p(l,"P",{"data-svelte-h":!0}),y(S)!=="svelte-ygg3x4"&&(S.innerHTML=Kl),ml=e(l),i(Q.$$.fragment,l),ul=e(l),A=p(l,"P",{"data-svelte-h":!0}),y(A)!=="svelte-7gpw7a"&&(A.textContent=Pl),dl=e(l),F=p(l,"P",{"data-svelte-h":!0}),y(F)!=="svelte-4fzrqm"&&(F.innerHTML=ls),bl=e(l),i(Y.$$.fragment,l),Il=e(l),X=p(l,"P",{"data-svelte-h":!0}),y(X)!=="svelte-1vx9tic"&&(X.innerHTML=ss),Cl=e(l),o=p(l,"DIV",{class:!0,"data-svelte-h":!0}),y(o)!=="svelte-1ct11n9"&&(o.innerHTML=as),Bl=e(l),W=p(l,"P",{"data-svelte-h":!0}),y(W)!=="svelte-1p4gwvr"&&(W.innerHTML=ns),gl=e(l),V=p(l,"P",{"data-svelte-h":!0}),y(V)!=="svelte-19d7o0j"&&(V.textContent=ts),Gl=e(l),i(E.$$.fragment,l),fl=e(l),z=p(l,"P",{"data-svelte-h":!0}),y(z)!=="svelte-q1u30o"&&(z.innerHTML=es),kl=e(l),i(v.$$.fragment,l),Zl=e(l),_=p(l,"P",{"data-svelte-h":!0}),y(_)!=="svelte-e2gcj5"&&(_.textContent=Ms),Nl=e(l),i(H.$$.fragment,l),Rl=e(l),i(q.$$.fragment,l),Sl=e(l),x=p(l,"P",{"data-svelte-h":!0}),y(x)!=="svelte-nrb4bc"&&(x.textContent=ps),Ql=e(l),D=p(l,"P",{"data-svelte-h":!0}),y(D)!=="svelte-168mt1p"&&(D.innerHTML=ys),Al=e(l),i(O.$$.fragment,l),Fl=e(l),L=p(l,"P",{"data-svelte-h":!0}),y(L)!=="svelte-1d35p3o"&&(L.innerHTML=Us),Yl=e(l),$=p(l,"P",{"data-svelte-h":!0}),y($)!=="svelte-wkzo7"&&($.innerHTML=is),Xl=e(l),i(K.$$.fragment,l),Wl=e(l),w=p(l,"DIV",{class:!0,"data-svelte-h":!0}),y(w)!=="svelte-tmflad"&&(w.innerHTML=cs),Vl=e(l),i(P.$$.fragment,l),El=e(l),sl=p(l,"P",{}),Js(sl).forEach(a),this.h()},h(){vl(j,"name","hf:doc:metadata"),vl(j,"content",bs),vl(o,"class","flex justify-center"),vl(w,"class","flex justify-center")},m(l,s){ms(document.head,j),n(l,nl,s),n(l,ll,s),n(l,tl,s),c(m,l,s),n(l,el,s),n(l,u,s),n(l,Ml,s),n(l,d,s),n(l,pl,s),c(b,l,s),n(l,yl,s),n(l,I,s),n(l,Ul,s),c(C,l,s),n(l,il,s),n(l,B,s),n(l,cl,s),c(g,l,s),n(l,Jl,s),c(G,l,s),n(l,Tl,s),n(l,f,s),n(l,rl,s),n(l,k,s),n(l,hl,s),c(Z,l,s),n(l,jl,s),n(l,N,s),n(l,ol,s),c(R,l,s),n(l,wl,s),n(l,S,s),n(l,ml,s),c(Q,l,s),n(l,ul,s),n(l,A,s),n(l,dl,s),n(l,F,s),n(l,bl,s),c(Y,l,s),n(l,Il,s),n(l,X,s),n(l,Cl,s),n(l,o,s),n(l,Bl,s),n(l,W,s),n(l,gl,s),n(l,V,s),n(l,Gl,s),c(E,l,s),n(l,fl,s),n(l,z,s),n(l,kl,s),c(v,l,s),n(l,Zl,s),n(l,_,s),n(l,Nl,s),c(H,l,s),n(l,Rl,s),c(q,l,s),n(l,Sl,s),n(l,x,s),n(l,Ql,s),n(l,D,s),n(l,Al,s),c(O,l,s),n(l,Fl,s),n(l,L,s),n(l,Yl,s),n(l,$,s),n(l,Xl,s),c(K,l,s),n(l,Wl,s),n(l,w,s),n(l,Vl,s),c(P,l,s),n(l,El,s),n(l,sl,s),zl=!0},p:rs,i(l){zl||(J(m.$$.fragment,l),J(b.$$.fragment,l),J(C.$$.fragment,l),J(g.$$.fragment,l),J(G.$$.fragment,l),J(Z.$$.fragment,l),J(R.$$.fragment,l),J(Q.$$.fragment,l),J(Y.$$.fragment,l),J(E.$$.fragment,l),J(v.$$.fragment,l),J(H.$$.fragment,l),J(q.$$.fragment,l),J(O.$$.fragment,l),J(K.$$.fragment,l),J(P.$$.fragment,l),zl=!0)},o(l){T(m.$$.fragment,l),T(b.$$.fragment,l),T(C.$$.fragment,l),T(g.$$.fragment,l),T(G.$$.fragment,l),T(Z.$$.fragment,l),T(R.$$.fragment,l),T(Q.$$.fragment,l),T(Y.$$.fragment,l),T(E.$$.fragment,l),T(v.$$.fragment,l),T(H.$$.fragment,l),T(q.$$.fragment,l),T(O.$$.fragment,l),T(K.$$.fragment,l),T(P.$$.fragment,l),zl=!1},d(l){l&&(a(nl),a(ll),a(tl),a(el),a(u),a(Ml),a(d),a(pl),a(yl),a(I),a(Ul),a(il),a(B),a(cl),a(Jl),a(Tl),a(f),a(rl),a(k),a(hl),a(jl),a(N),a(ol),a(wl),a(S),a(ml),a(ul),a(A),a(dl),a(F),a(bl),a(Il),a(X),a(Cl),a(o),a(Bl),a(W),a(gl),a(V),a(Gl),a(fl),a(z),a(kl),a(Zl),a(_),a(Nl),a(Rl),a(Sl),a(x),a(Ql),a(D),a(Al),a(Fl),a(L),a(Yl),a($),a(Xl),a(Wl),a(w),a(Vl),a(El),a(sl)),a(j),r(m,l),r(b,l),r(C,l),r(g,l),r(G,l),r(Z,l),r(R,l),r(Q,l),r(Y,l),r(E,l),r(v,l),r(H,l),r(q,l),r(O,l),r(K,l),r(P,l)}}}const bs='{"title":"Spark","local":"spark","sections":[{"title":"Installation","local":"installation","sections":[],"depth":2},{"title":"Authentication","local":"authentication","sections":[],"depth":2},{"title":"Read","local":"read","sections":[],"depth":2},{"title":"Write","local":"write","sections":[],"depth":2}],"depth":1}';function Is(_l){return hs(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class fs extends js{constructor(j){super(),os(this,j,Is,ds,Ts,{})}}export{fs as component};
