import{s as pt,n as it,o as ct}from"../chunks/scheduler.bdbef820.js";import{S as rt,i as ot,g as p,s as l,r,A as dt,h as i,f as a,c as n,j as nt,u as o,x as c,k as J,y as mt,a as e,v as d,d as m,t as h,w as u}from"../chunks/index.c0aea24a.js";import{C as g}from"../chunks/CodeBlock.6ccca92e.js";import{H as vs,E as ht}from"../chunks/EditOnGithub.725ee0c1.js";function ut(Vs){let M,ts,O,as,U,es,X,Qs='<a href="https://github.com/facebookresearch/faiss" rel="nofollow">FAISS</a> and <a href="https://www.elastic.co/elasticsearch/" rel="nofollow">Elasticsearch</a> enables searching for examples in a dataset. This can be useful when you want to retrieve specific examples from a dataset that are relevant to your NLP task. For example, if you are working on a Open Domain Question Answering task, you may want to only return examples that are relevant to answering your question.',ls,k,zs="This guide will show you how to build an index for your dataset that will allow you to search it.",ns,W,ps,q,Es='FAISS retrieves documents based on the similarity of their vector representations. In this example, you will generate the vector representations with the <a href="https://huggingface.co/transformers/model_doc/dpr.html" rel="nofollow">DPR</a> model.',is,R,Ys="<li>Download the DPR model from 🤗 Transformers:</li>",cs,$,rs,j,Bs="<li>Load your dataset and compute the vector representations:</li>",os,F,ds,y,Hs='<li>Create the index with <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset.add_faiss_index">Dataset.add_faiss_index()</a>:</li>',ms,N,hs,f,Ls='<li>Now you can query your dataset with the <code>embeddings</code> index. Load the DPR Question Encoder, and search for a question with <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset.get_nearest_examples">Dataset.get_nearest_examples()</a>:</li>',us,G,gs,T,Ds='<li>You can access the index with <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset.get_index">Dataset.get_index()</a> and use it for special operations, e.g. query it using <code>range_search</code>:</li>',Js,C,Ms,w,Ss='<li>When you are done querying, save the index on disk with <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset.save_faiss_index">Dataset.save_faiss_index()</a>:</li>',js,v,ys,_,As='<li>Reload it at a later time with <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset.load_faiss_index">Dataset.load_faiss_index()</a>:</li>',fs,V,Ts,Q,ws,z,Ps="Unlike FAISS, Elasticsearch retrieves documents based on exact matches.",_s,E,Ks='Start Elasticsearch on your machine, or see the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/setup.html" rel="nofollow">Elasticsearch installation guide</a> if you don’t already have it installed.',bs,Y,Os="<li>Load the dataset you want to index:</li>",xs,B,Zs,b,st='<li>Build the index with <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset.add_elasticsearch_index">Dataset.add_elasticsearch_index()</a>:</li>',Is,H,Us,x,tt='<li>Then you can query the <code>context</code> index with <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset.get_nearest_examples">Dataset.get_nearest_examples()</a>:</li>',Xs,L,ks,Z,at="<li>If you want to reuse the index, define the <code>es_index_name</code> parameter when you build the index:</li>",Ws,D,qs,I,et='<li>Reload it later with the index name when you call <a href="/docs/datasets/pr_6981/en/package_reference/main_classes#datasets.Dataset.load_elasticsearch_index">Dataset.load_elasticsearch_index()</a>:</li>',Rs,S,$s,A,lt="For more advanced Elasticsearch usage, you can specify your own configuration with custom settings:",Fs,P,Ns,K,Gs,ss,Cs;return U=new vs({props:{title:"Search index",local:"search-index",headingTag:"h1"}}),W=new vs({props:{title:"FAISS",local:"faiss",headingTag:"h2"}}),$=new g({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERQUkNvbnRleHRFbmNvZGVyJTJDJTIwRFBSQ29udGV4dEVuY29kZXJUb2tlbml6ZXIlMEFpbXBvcnQlMjB0b3JjaCUwQXRvcmNoLnNldF9ncmFkX2VuYWJsZWQoRmFsc2UpJTBBY3R4X2VuY29kZXIlMjAlM0QlMjBEUFJDb250ZXh0RW5jb2Rlci5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZkcHItY3R4X2VuY29kZXItc2luZ2xlLW5xLWJhc2UlMjIpJTBBY3R4X3Rva2VuaXplciUyMCUzRCUyMERQUkNvbnRleHRFbmNvZGVyVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRmRwci1jdHhfZW5jb2Rlci1zaW5nbGUtbnEtYmFzZSUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DPRContextEncoder, DPRContextEncoderTokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span>torch.set_grad_enabled(<span class="hljs-literal">False</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ctx_encoder = DPRContextEncoder.from_pretrained(<span class="hljs-string">&quot;facebook/dpr-ctx_encoder-single-nq-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ctx_tokenizer = DPRContextEncoderTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/dpr-ctx_encoder-single-nq-base&quot;</span>)`,wrap:!1}}),F=new g({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJ2NyaW1lX2FuZF9wdW5pc2gnJTJDJTIwc3BsaXQlM0QndHJhaW4lNUIlM0ExMDAlNUQnKSUwQWRzX3dpdGhfZW1iZWRkaW5ncyUyMCUzRCUyMGRzLm1hcChsYW1iZGElMjBleGFtcGxlJTNBJTIwJTdCJ2VtYmVkZGluZ3MnJTNBJTIwY3R4X2VuY29kZXIoKipjdHhfdG9rZW5pemVyKGV4YW1wbGUlNUIlMjJsaW5lJTIyJTVEJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikpJTVCMCU1RCU1QjAlNUQubnVtcHkoKSU3RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = load_dataset(<span class="hljs-string">&#x27;crime_and_punish&#x27;</span>, split=<span class="hljs-string">&#x27;train[:100]&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds_with_embeddings = ds.<span class="hljs-built_in">map</span>(<span class="hljs-keyword">lambda</span> example: {<span class="hljs-string">&#x27;embeddings&#x27;</span>: ctx_encoder(**ctx_tokenizer(example[<span class="hljs-string">&quot;line&quot;</span>], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>))[<span class="hljs-number">0</span>][<span class="hljs-number">0</span>].numpy()})`,wrap:!1}}),N=new g({props:{code:"ZHNfd2l0aF9lbWJlZGRpbmdzLmFkZF9mYWlzc19pbmRleChjb2x1bW4lM0QnZW1iZWRkaW5ncycp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>ds_with_embeddings.add_faiss_index(column=<span class="hljs-string">&#x27;embeddings&#x27;</span>)',wrap:!1}}),G=new g({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERQUlF1ZXN0aW9uRW5jb2RlciUyQyUyMERQUlF1ZXN0aW9uRW5jb2RlclRva2VuaXplciUwQXFfZW5jb2RlciUyMCUzRCUyMERQUlF1ZXN0aW9uRW5jb2Rlci5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZkcHItcXVlc3Rpb25fZW5jb2Rlci1zaW5nbGUtbnEtYmFzZSUyMiklMEFxX3Rva2VuaXplciUyMCUzRCUyMERQUlF1ZXN0aW9uRW5jb2RlclRva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZkcHItcXVlc3Rpb25fZW5jb2Rlci1zaW5nbGUtbnEtYmFzZSUyMiklMEElMEFxdWVzdGlvbiUyMCUzRCUyMCUyMklzJTIwaXQlMjBzZXJpb3VzJTIwJTNGJTIyJTBBcXVlc3Rpb25fZW1iZWRkaW5nJTIwJTNEJTIwcV9lbmNvZGVyKCoqcV90b2tlbml6ZXIocXVlc3Rpb24lMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSklNUIwJTVEJTVCMCU1RC5udW1weSgpJTBBc2NvcmVzJTJDJTIwcmV0cmlldmVkX2V4YW1wbGVzJTIwJTNEJTIwZHNfd2l0aF9lbWJlZGRpbmdzLmdldF9uZWFyZXN0X2V4YW1wbGVzKCdlbWJlZGRpbmdzJyUyQyUyMHF1ZXN0aW9uX2VtYmVkZGluZyUyQyUyMGslM0QxMCklMEFyZXRyaWV2ZWRfZXhhbXBsZXMlNUIlMjJsaW5lJTIyJTVEJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DPRQuestionEncoder, DPRQuestionEncoderTokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span>q_encoder = DPRQuestionEncoder.from_pretrained(<span class="hljs-string">&quot;facebook/dpr-question_encoder-single-nq-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>q_tokenizer = DPRQuestionEncoderTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/dpr-question_encoder-single-nq-base&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>question = <span class="hljs-string">&quot;Is it serious ?&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>question_embedding = q_encoder(**q_tokenizer(question, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>))[<span class="hljs-number">0</span>][<span class="hljs-number">0</span>].numpy()
<span class="hljs-meta">&gt;&gt;&gt; </span>scores, retrieved_examples = ds_with_embeddings.get_nearest_examples(<span class="hljs-string">&#x27;embeddings&#x27;</span>, question_embedding, k=<span class="hljs-number">10</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>retrieved_examples[<span class="hljs-string">&quot;line&quot;</span>][<span class="hljs-number">0</span>]
<span class="hljs-string">&#x27;_that_ serious? It is not serious at all. It’s simply a fantasy to amuse\\r\\n&#x27;</span>`,wrap:!1}}),C=new g({props:{code:"ZmFpc3NfaW5kZXglMjAlM0QlMjBkc193aXRoX2VtYmVkZGluZ3MuZ2V0X2luZGV4KCdlbWJlZGRpbmdzJykuZmFpc3NfaW5kZXglMEFsaW1pdHMlMkMlMjBkaXN0YW5jZXMlMkMlMjBpbmRpY2VzJTIwJTNEJTIwZmFpc3NfaW5kZXgucmFuZ2Vfc2VhcmNoKHglM0RxdWVzdGlvbl9lbWJlZGRpbmcucmVzaGFwZSgxJTJDJTIwLTEpJTJDJTIwdGhyZXNoJTNEMC45NSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>faiss_index = ds_with_embeddings.get_index(<span class="hljs-string">&#x27;embeddings&#x27;</span>).faiss_index
<span class="hljs-meta">&gt;&gt;&gt; </span>limits, distances, indices = faiss_index.range_search(x=question_embedding.reshape(<span class="hljs-number">1</span>, -<span class="hljs-number">1</span>), thresh=<span class="hljs-number">0.95</span>)`,wrap:!1}}),v=new g({props:{code:"ZHNfd2l0aF9lbWJlZGRpbmdzLnNhdmVfZmFpc3NfaW5kZXgoJ2VtYmVkZGluZ3MnJTJDJTIwJ215X2luZGV4LmZhaXNzJyk=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>ds_with_embeddings.save_faiss_index(<span class="hljs-string">&#x27;embeddings&#x27;</span>, <span class="hljs-string">&#x27;my_index.faiss&#x27;</span>)',wrap:!1}}),V=new g({props:{code:"ZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJ2NyaW1lX2FuZF9wdW5pc2gnJTJDJTIwc3BsaXQlM0QndHJhaW4lNUIlM0ExMDAlNUQnKSUwQWRzLmxvYWRfZmFpc3NfaW5kZXgoJ2VtYmVkZGluZ3MnJTJDJTIwJ215X2luZGV4LmZhaXNzJyk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>ds = load_dataset(<span class="hljs-string">&#x27;crime_and_punish&#x27;</span>, split=<span class="hljs-string">&#x27;train[:100]&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds.load_faiss_index(<span class="hljs-string">&#x27;embeddings&#x27;</span>, <span class="hljs-string">&#x27;my_index.faiss&#x27;</span>)`,wrap:!1}}),Q=new vs({props:{title:"Elasticsearch",local:"elasticsearch",headingTag:"h2"}}),B=new g({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBc3F1YWQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJ3NxdWFkJyUyQyUyMHNwbGl0JTNEJ3ZhbGlkYXRpb24nKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>squad = load_dataset(<span class="hljs-string">&#x27;squad&#x27;</span>, split=<span class="hljs-string">&#x27;validation&#x27;</span>)`,wrap:!1}}),H=new g({props:{code:"c3F1YWQuYWRkX2VsYXN0aWNzZWFyY2hfaW5kZXgoJTIyY29udGV4dCUyMiUyQyUyMGhvc3QlM0QlMjJsb2NhbGhvc3QlMjIlMkMlMjBwb3J0JTNEJTIyOTIwMCUyMik=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>squad.add_elasticsearch_index(<span class="hljs-string">&quot;context&quot;</span>, host=<span class="hljs-string">&quot;localhost&quot;</span>, port=<span class="hljs-string">&quot;9200&quot;</span>)',wrap:!1}}),L=new g({props:{code:"cXVlcnklMjAlM0QlMjAlMjJtYWNoaW5lJTIyJTBBc2NvcmVzJTJDJTIwcmV0cmlldmVkX2V4YW1wbGVzJTIwJTNEJTIwc3F1YWQuZ2V0X25lYXJlc3RfZXhhbXBsZXMoJTIyY29udGV4dCUyMiUyQyUyMHF1ZXJ5JTJDJTIwayUzRDEwKSUwQXJldHJpZXZlZF9leGFtcGxlcyU1QiUyMnRpdGxlJTIyJTVEJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>query = <span class="hljs-string">&quot;machine&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>scores, retrieved_examples = squad.get_nearest_examples(<span class="hljs-string">&quot;context&quot;</span>, query, k=<span class="hljs-number">10</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>retrieved_examples[<span class="hljs-string">&quot;title&quot;</span>][<span class="hljs-number">0</span>]
<span class="hljs-string">&#x27;Computational_complexity_theory&#x27;</span>`,wrap:!1}}),D=new g({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBc3F1YWQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJ3NxdWFkJyUyQyUyMHNwbGl0JTNEJ3ZhbGlkYXRpb24nKSUwQXNxdWFkLmFkZF9lbGFzdGljc2VhcmNoX2luZGV4KCUyMmNvbnRleHQlMjIlMkMlMjBob3N0JTNEJTIybG9jYWxob3N0JTIyJTJDJTIwcG9ydCUzRCUyMjkyMDAlMjIlMkMlMjBlc19pbmRleF9uYW1lJTNEJTIyaGZfc3F1YWRfdmFsX2NvbnRleHQlMjIpJTBBc3F1YWQuZ2V0X2luZGV4KCUyMmNvbnRleHQlMjIpLmVzX2luZGV4X25hbWU=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>squad = load_dataset(<span class="hljs-string">&#x27;squad&#x27;</span>, split=<span class="hljs-string">&#x27;validation&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>squad.add_elasticsearch_index(<span class="hljs-string">&quot;context&quot;</span>, host=<span class="hljs-string">&quot;localhost&quot;</span>, port=<span class="hljs-string">&quot;9200&quot;</span>, es_index_name=<span class="hljs-string">&quot;hf_squad_val_context&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>squad.get_index(<span class="hljs-string">&quot;context&quot;</span>).es_index_name
hf_squad_val_context`,wrap:!1}}),S=new g({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBc3F1YWQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJ3NxdWFkJyUyQyUyMHNwbGl0JTNEJ3ZhbGlkYXRpb24nKSUwQXNxdWFkLmxvYWRfZWxhc3RpY3NlYXJjaF9pbmRleCglMjJjb250ZXh0JTIyJTJDJTIwaG9zdCUzRCUyMmxvY2FsaG9zdCUyMiUyQyUyMHBvcnQlM0QlMjI5MjAwJTIyJTJDJTIwZXNfaW5kZXhfbmFtZSUzRCUyMmhmX3NxdWFkX3ZhbF9jb250ZXh0JTIyKSUwQXF1ZXJ5JTIwJTNEJTIwJTIybWFjaGluZSUyMiUwQXNjb3JlcyUyQyUyMHJldHJpZXZlZF9leGFtcGxlcyUyMCUzRCUyMHNxdWFkLmdldF9uZWFyZXN0X2V4YW1wbGVzKCUyMmNvbnRleHQlMjIlMkMlMjBxdWVyeSUyQyUyMGslM0QxMCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>squad = load_dataset(<span class="hljs-string">&#x27;squad&#x27;</span>, split=<span class="hljs-string">&#x27;validation&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>squad.load_elasticsearch_index(<span class="hljs-string">&quot;context&quot;</span>, host=<span class="hljs-string">&quot;localhost&quot;</span>, port=<span class="hljs-string">&quot;9200&quot;</span>, es_index_name=<span class="hljs-string">&quot;hf_squad_val_context&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>query = <span class="hljs-string">&quot;machine&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>scores, retrieved_examples = squad.get_nearest_examples(<span class="hljs-string">&quot;context&quot;</span>, query, k=<span class="hljs-number">10</span>)`,wrap:!1}}),P=new g({props:{code:"aW1wb3J0JTIwZWxhc3RpY3NlYXJjaCUyMGFzJTIwZXMlMEFpbXBvcnQlMjBlbGFzdGljc2VhcmNoLmhlbHBlcnMlMEFmcm9tJTIwZWxhc3RpY3NlYXJjaCUyMGltcG9ydCUyMEVsYXN0aWNzZWFyY2glMEFlc19jbGllbnQlMjAlM0QlMjBFbGFzdGljc2VhcmNoKCU1QiU3QiUyMmhvc3QlMjIlM0ElMjAlMjJsb2NhbGhvc3QlMjIlMkMlMjAlMjJwb3J0JTIyJTNBJTIwJTIyOTIwMCUyMiU3RCU1RCklMjAlMjAlMjMlMjBkZWZhdWx0JTIwY2xpZW50JTBBZXNfY29uZmlnJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIyc2V0dGluZ3MlMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJudW1iZXJfb2Zfc2hhcmRzJTIyJTNBJTIwMSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmFuYWx5c2lzJTIyJTNBJTIwJTdCJTIyYW5hbHl6ZXIlMjIlM0ElMjAlN0IlMjJzdG9wX3N0YW5kYXJkJTIyJTNBJTIwJTdCJTIydHlwZSUyMiUzQSUyMCUyMnN0YW5kYXJkJTIyJTJDJTIwJTIyJTIwc3RvcHdvcmRzJTIyJTNBJTIwJTIyX2VuZ2xpc2hfJTIyJTdEJTdEJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIybWFwcGluZ3MlMjIlM0ElMjAlN0IlMjJwcm9wZXJ0aWVzJTIyJTNBJTIwJTdCJTIydGV4dCUyMiUzQSUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIyYW5hbHl6ZXIlMjIlM0ElMjAlMjJzdGFuZGFyZCUyMiUyQyUyMCUyMnNpbWlsYXJpdHklMjIlM0ElMjAlMjJCTTI1JTIyJTdEJTdEJTdEJTJDJTBBJTdEJTIwJTIwJTIzJTIwZGVmYXVsdCUyMGNvbmZpZyUwQWVzX2luZGV4X25hbWUlMjAlM0QlMjAlMjJoZl9zcXVhZF9jb250ZXh0JTIyJTIwJTIwJTIzJTIwbmFtZSUyMG9mJTIwdGhlJTIwaW5kZXglMjBpbiUyMEVsYXN0aWNzZWFyY2glMEFzcXVhZC5hZGRfZWxhc3RpY3NlYXJjaF9pbmRleCglMjJjb250ZXh0JTIyJTJDJTIwZXNfY2xpZW50JTNEZXNfY2xpZW50JTJDJTIwZXNfY29uZmlnJTNEZXNfY29uZmlnJTJDJTIwZXNfaW5kZXhfbmFtZSUzRGVzX2luZGV4X25hbWUp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> elasticsearch <span class="hljs-keyword">as</span> es
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> elasticsearch.helpers
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> elasticsearch <span class="hljs-keyword">import</span> Elasticsearch
<span class="hljs-meta">&gt;&gt;&gt; </span>es_client = Elasticsearch([{<span class="hljs-string">&quot;host&quot;</span>: <span class="hljs-string">&quot;localhost&quot;</span>, <span class="hljs-string">&quot;port&quot;</span>: <span class="hljs-string">&quot;9200&quot;</span>}])  <span class="hljs-comment"># default client</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>es_config = {
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;settings&quot;</span>: {
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;number_of_shards&quot;</span>: <span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;analysis&quot;</span>: {<span class="hljs-string">&quot;analyzer&quot;</span>: {<span class="hljs-string">&quot;stop_standard&quot;</span>: {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;standard&quot;</span>, <span class="hljs-string">&quot; stopwords&quot;</span>: <span class="hljs-string">&quot;_english_&quot;</span>}}},
<span class="hljs-meta">... </span>    },
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;mappings&quot;</span>: {<span class="hljs-string">&quot;properties&quot;</span>: {<span class="hljs-string">&quot;text&quot;</span>: {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;analyzer&quot;</span>: <span class="hljs-string">&quot;standard&quot;</span>, <span class="hljs-string">&quot;similarity&quot;</span>: <span class="hljs-string">&quot;BM25&quot;</span>}}},
<span class="hljs-meta">... </span>}  <span class="hljs-comment"># default config</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>es_index_name = <span class="hljs-string">&quot;hf_squad_context&quot;</span>  <span class="hljs-comment"># name of the index in Elasticsearch</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>squad.add_elasticsearch_index(<span class="hljs-string">&quot;context&quot;</span>, es_client=es_client, es_config=es_config, es_index_name=es_index_name)`,wrap:!1}}),K=new ht({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/faiss_es.mdx"}}),{c(){M=p("meta"),ts=l(),O=p("p"),as=l(),r(U.$$.fragment),es=l(),X=p("p"),X.innerHTML=Qs,ls=l(),k=p("p"),k.textContent=zs,ns=l(),r(W.$$.fragment),ps=l(),q=p("p"),q.innerHTML=Es,is=l(),R=p("ol"),R.innerHTML=Ys,cs=l(),r($.$$.fragment),rs=l(),j=p("ol"),j.innerHTML=Bs,os=l(),r(F.$$.fragment),ds=l(),y=p("ol"),y.innerHTML=Hs,ms=l(),r(N.$$.fragment),hs=l(),f=p("ol"),f.innerHTML=Ls,us=l(),r(G.$$.fragment),gs=l(),T=p("ol"),T.innerHTML=Ds,Js=l(),r(C.$$.fragment),Ms=l(),w=p("ol"),w.innerHTML=Ss,js=l(),r(v.$$.fragment),ys=l(),_=p("ol"),_.innerHTML=As,fs=l(),r(V.$$.fragment),Ts=l(),r(Q.$$.fragment),ws=l(),z=p("p"),z.textContent=Ps,_s=l(),E=p("p"),E.innerHTML=Ks,bs=l(),Y=p("ol"),Y.innerHTML=Os,xs=l(),r(B.$$.fragment),Zs=l(),b=p("ol"),b.innerHTML=st,Is=l(),r(H.$$.fragment),Us=l(),x=p("ol"),x.innerHTML=tt,Xs=l(),r(L.$$.fragment),ks=l(),Z=p("ol"),Z.innerHTML=at,Ws=l(),r(D.$$.fragment),qs=l(),I=p("ol"),I.innerHTML=et,Rs=l(),r(S.$$.fragment),$s=l(),A=p("p"),A.textContent=lt,Fs=l(),r(P.$$.fragment),Ns=l(),r(K.$$.fragment),Gs=l(),ss=p("p"),this.h()},l(s){const t=dt("svelte-u9bgzb",document.head);M=i(t,"META",{name:!0,content:!0}),t.forEach(a),ts=n(s),O=i(s,"P",{}),nt(O).forEach(a),as=n(s),o(U.$$.fragment,s),es=n(s),X=i(s,"P",{"data-svelte-h":!0}),c(X)!=="svelte-5c7wnl"&&(X.innerHTML=Qs),ls=n(s),k=i(s,"P",{"data-svelte-h":!0}),c(k)!=="svelte-hdwpul"&&(k.textContent=zs),ns=n(s),o(W.$$.fragment,s),ps=n(s),q=i(s,"P",{"data-svelte-h":!0}),c(q)!=="svelte-se9cy2"&&(q.innerHTML=Es),is=n(s),R=i(s,"OL",{"data-svelte-h":!0}),c(R)!=="svelte-x1ghuo"&&(R.innerHTML=Ys),cs=n(s),o($.$$.fragment,s),rs=n(s),j=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(j)!=="svelte-1teerx9"&&(j.innerHTML=Bs),os=n(s),o(F.$$.fragment,s),ds=n(s),y=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(y)!=="svelte-nmuqpo"&&(y.innerHTML=Hs),ms=n(s),o(N.$$.fragment,s),hs=n(s),f=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(f)!=="svelte-6z791m"&&(f.innerHTML=Ls),us=n(s),o(G.$$.fragment,s),gs=n(s),T=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(T)!=="svelte-djas5e"&&(T.innerHTML=Ds),Js=n(s),o(C.$$.fragment,s),Ms=n(s),w=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(w)!=="svelte-1ogrbjj"&&(w.innerHTML=Ss),js=n(s),o(v.$$.fragment,s),ys=n(s),_=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(_)!=="svelte-r7m33g"&&(_.innerHTML=As),fs=n(s),o(V.$$.fragment,s),Ts=n(s),o(Q.$$.fragment,s),ws=n(s),z=i(s,"P",{"data-svelte-h":!0}),c(z)!=="svelte-8ul8ma"&&(z.textContent=Ps),_s=n(s),E=i(s,"P",{"data-svelte-h":!0}),c(E)!=="svelte-195ztu1"&&(E.innerHTML=Ks),bs=n(s),Y=i(s,"OL",{"data-svelte-h":!0}),c(Y)!=="svelte-3fd7ny"&&(Y.innerHTML=Os),xs=n(s),o(B.$$.fragment,s),Zs=n(s),b=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(b)!=="svelte-13u991b"&&(b.innerHTML=st),Is=n(s),o(H.$$.fragment,s),Us=n(s),x=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(x)!=="svelte-lpwzlo"&&(x.innerHTML=tt),Xs=n(s),o(L.$$.fragment,s),ks=n(s),Z=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(Z)!=="svelte-1tchn11"&&(Z.innerHTML=at),Ws=n(s),o(D.$$.fragment,s),qs=n(s),I=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(I)!=="svelte-1w0woiu"&&(I.innerHTML=et),Rs=n(s),o(S.$$.fragment,s),$s=n(s),A=i(s,"P",{"data-svelte-h":!0}),c(A)!=="svelte-1ajeg13"&&(A.textContent=lt),Fs=n(s),o(P.$$.fragment,s),Ns=n(s),o(K.$$.fragment,s),Gs=n(s),ss=i(s,"P",{}),nt(ss).forEach(a),this.h()},h(){J(M,"name","hf:doc:metadata"),J(M,"content",gt),J(j,"start","2"),J(y,"start","3"),J(f,"start","4"),J(T,"start","5"),J(w,"start","6"),J(_,"start","7"),J(b,"start","2"),J(x,"start","3"),J(Z,"start","4"),J(I,"start","5")},m(s,t){mt(document.head,M),e(s,ts,t),e(s,O,t),e(s,as,t),d(U,s,t),e(s,es,t),e(s,X,t),e(s,ls,t),e(s,k,t),e(s,ns,t),d(W,s,t),e(s,ps,t),e(s,q,t),e(s,is,t),e(s,R,t),e(s,cs,t),d($,s,t),e(s,rs,t),e(s,j,t),e(s,os,t),d(F,s,t),e(s,ds,t),e(s,y,t),e(s,ms,t),d(N,s,t),e(s,hs,t),e(s,f,t),e(s,us,t),d(G,s,t),e(s,gs,t),e(s,T,t),e(s,Js,t),d(C,s,t),e(s,Ms,t),e(s,w,t),e(s,js,t),d(v,s,t),e(s,ys,t),e(s,_,t),e(s,fs,t),d(V,s,t),e(s,Ts,t),d(Q,s,t),e(s,ws,t),e(s,z,t),e(s,_s,t),e(s,E,t),e(s,bs,t),e(s,Y,t),e(s,xs,t),d(B,s,t),e(s,Zs,t),e(s,b,t),e(s,Is,t),d(H,s,t),e(s,Us,t),e(s,x,t),e(s,Xs,t),d(L,s,t),e(s,ks,t),e(s,Z,t),e(s,Ws,t),d(D,s,t),e(s,qs,t),e(s,I,t),e(s,Rs,t),d(S,s,t),e(s,$s,t),e(s,A,t),e(s,Fs,t),d(P,s,t),e(s,Ns,t),d(K,s,t),e(s,Gs,t),e(s,ss,t),Cs=!0},p:it,i(s){Cs||(m(U.$$.fragment,s),m(W.$$.fragment,s),m($.$$.fragment,s),m(F.$$.fragment,s),m(N.$$.fragment,s),m(G.$$.fragment,s),m(C.$$.fragment,s),m(v.$$.fragment,s),m(V.$$.fragment,s),m(Q.$$.fragment,s),m(B.$$.fragment,s),m(H.$$.fragment,s),m(L.$$.fragment,s),m(D.$$.fragment,s),m(S.$$.fragment,s),m(P.$$.fragment,s),m(K.$$.fragment,s),Cs=!0)},o(s){h(U.$$.fragment,s),h(W.$$.fragment,s),h($.$$.fragment,s),h(F.$$.fragment,s),h(N.$$.fragment,s),h(G.$$.fragment,s),h(C.$$.fragment,s),h(v.$$.fragment,s),h(V.$$.fragment,s),h(Q.$$.fragment,s),h(B.$$.fragment,s),h(H.$$.fragment,s),h(L.$$.fragment,s),h(D.$$.fragment,s),h(S.$$.fragment,s),h(P.$$.fragment,s),h(K.$$.fragment,s),Cs=!1},d(s){s&&(a(ts),a(O),a(as),a(es),a(X),a(ls),a(k),a(ns),a(ps),a(q),a(is),a(R),a(cs),a(rs),a(j),a(os),a(ds),a(y),a(ms),a(hs),a(f),a(us),a(gs),a(T),a(Js),a(Ms),a(w),a(js),a(ys),a(_),a(fs),a(Ts),a(ws),a(z),a(_s),a(E),a(bs),a(Y),a(xs),a(Zs),a(b),a(Is),a(Us),a(x),a(Xs),a(ks),a(Z),a(Ws),a(qs),a(I),a(Rs),a($s),a(A),a(Fs),a(Ns),a(Gs),a(ss)),a(M),u(U,s),u(W,s),u($,s),u(F,s),u(N,s),u(G,s),u(C,s),u(v,s),u(V,s),u(Q,s),u(B,s),u(H,s),u(L,s),u(D,s),u(S,s),u(P,s),u(K,s)}}}const gt='{"title":"Search index","local":"search-index","sections":[{"title":"FAISS","local":"faiss","sections":[],"depth":2},{"title":"Elasticsearch","local":"elasticsearch","sections":[],"depth":2}],"depth":1}';function Jt(Vs){return ct(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Tt extends rt{constructor(M){super(),ot(this,M,Jt,ut,pt,{})}}export{Tt as component};
