import{s as Dl,o as Ll,n as Hl}from"../chunks/scheduler.37c15a92.js";import{S as Pl,i as Kl,g as f,s as r,r as m,A as Ol,h as T,f as l,c,j as El,u as U,x as w,k as De,l as il,y as es,a as s,v as y,t as M,b as el,d as p,w as J,p as tl}from"../chunks/index.2bf4358c.js";import{T as Al}from"../chunks/Tip.363c041f.js";import{Y as ts}from"../chunks/Youtube.1e50a667.js";import{C as j}from"../chunks/CodeBlock.4e987730.js";import{C as ql}from"../chunks/CourseFloatingBanner.6add7356.js";import{F as ls}from"../chunks/FrameworkSwitchCourse.8d4d4ab6.js";import{H as Le,E as ss}from"../chunks/getInferenceSnippets.8b9eb92a.js";function as(C){let a,o;return a=new ql({props:{chapter:5,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter5/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter5/section6_tf.ipynb"}]}}),{c(){m(a.$$.fragment)},l(n){U(a.$$.fragment,n)},m(n,h){y(a,n,h),o=!0},i(n){o||(p(a.$$.fragment,n),o=!0)},o(n){M(a.$$.fragment,n),o=!1},d(n){J(a,n)}}}function ns(C){let a,o;return a=new ql({props:{chapter:5,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter5/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter5/section6_pt.ipynb"}]}}),{c(){m(a.$$.fragment)},l(n){U(a.$$.fragment,n)},m(n,h){y(a,n,h),o=!0},i(n){o||(p(a.$$.fragment,n),o=!0)},o(n){M(a.$$.fragment,n),o=!1},d(n){J(a,n)}}}function is(C){let a,o='✏️ <strong>Încercați!</strong> Vezi dacă poți utiliza <code>Dataset.map()</code> pentru a exploda coloana <code>comments</code> din <code>issues_dataset</code> <em>fără</em> a recurge la utilizarea Pandas. Acest lucru este puțin dificil; s-ar putea să găsiți utilă secțiunea <a href="https://huggingface.co/docs/datasets/about_map_batch#batch-mapping" rel="nofollow">“Mapping batch”</a> din documentația 🤗 Datasets pentru această sarcină.';return{c(){a=f("p"),a.innerHTML=o},l(n){a=T(n,"P",{"data-svelte-h":!0}),w(a)!=="svelte-1nrl4qf"&&(a.innerHTML=o)},m(n,h){s(n,a,h)},p:Hl,d(n){n&&l(a)}}}function rs(C){let a,o,n,h="Notăm că am setat <code>from_pt=True</code> ca argument al metodei <code>from_pretrained()</code>. Acest lucru se datorează faptului că checkpoint-ul <code>multi-qa-mpnet-base-dot-v1</code> are doar PyTorch weights, astfel încât setarea <code>from_pt=True</code> le va converti automat în format TensorFlow pentru noi. După cum puteți vedea, este foarte simplu să treci de la un framework la altul în 🤗 Transformers!",u;return a=new j({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBURkF1dG9Nb2RlbCUwQSUwQW1vZGVsX2NrcHQlMjAlM0QlMjAlMjJzZW50ZW5jZS10cmFuc2Zvcm1lcnMlMkZtdWx0aS1xYS1tcG5ldC1iYXNlLWRvdC12MSUyMiUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2NrcHQpJTBBbW9kZWwlMjAlM0QlMjBURkF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQobW9kZWxfY2twdCUyQyUyMGZyb21fcHQlM0RUcnVlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModel

model_ckpt = <span class="hljs-string">&quot;sentence-transformers/multi-qa-mpnet-base-dot-v1&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = TFAutoModel.from_pretrained(model_ckpt, from_pt=<span class="hljs-literal">True</span>)`,wrap:!1}}),{c(){m(a.$$.fragment),o=r(),n=f("p"),n.innerHTML=h},l(d){U(a.$$.fragment,d),o=c(d),n=T(d,"P",{"data-svelte-h":!0}),w(n)!=="svelte-1wd48pf"&&(n.innerHTML=h)},m(d,G){y(a,d,G),s(d,o,G),s(d,n,G),u=!0},i(d){u||(p(a.$$.fragment,d),u=!0)},o(d){M(a.$$.fragment,d),u=!1},d(d){d&&(l(o),l(n)),J(a,d)}}}function cs(C){let a,o,n,h="Pentru a accelera procesul de embedding, este util să punem modelul și inputurile pe un GPU, deci hai să facem asta acum:",u,d,G;return a=new j({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWwlMEElMEFtb2RlbF9ja3B0JTIwJTNEJTIwJTIyc2VudGVuY2UtdHJhbnNmb3JtZXJzJTJGbXVsdGktcWEtbXBuZXQtYmFzZS1kb3QtdjElMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9ja3B0KSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZChtb2RlbF9ja3B0KQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModel

model_ckpt = <span class="hljs-string">&quot;sentence-transformers/multi-qa-mpnet-base-dot-v1&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = AutoModel.from_pretrained(model_ckpt)`,wrap:!1}}),d=new j({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEFkZXZpY2UlMjAlM0QlMjB0b3JjaC5kZXZpY2UoJTIyY3VkYSUyMiklMEFtb2RlbC50byhkZXZpY2Up",highlighted:`<span class="hljs-keyword">import</span> torch

device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>)
model.to(device)`,wrap:!1}}),{c(){m(a.$$.fragment),o=r(),n=f("p"),n.textContent=h,u=r(),m(d.$$.fragment)},l(b){U(a.$$.fragment,b),o=c(b),n=T(b,"P",{"data-svelte-h":!0}),w(n)!=="svelte-1y5lz39"&&(n.textContent=h),u=c(b),U(d.$$.fragment,b)},m(b,Q){y(a,b,Q),s(b,o,Q),s(b,n,Q),s(b,u,Q),y(d,b,Q),G=!0},i(b){G||(p(a.$$.fragment,b),p(d.$$.fragment,b),G=!0)},o(b){M(a.$$.fragment,b),M(d.$$.fragment,b),G=!1},d(b){b&&(l(o),l(n),l(u)),J(a,b),J(d,b)}}}function us(C){let a,o,n,h="Putem testa funcția oferindui prima intrare de text în corpusul nostru și analizând output shapeul:",u,d,G,b,Q,Z,R="Minunat, am transformat prima intrare din corpusul nostru într-un vector de 768 de dimensiuni! Putem utiliza <code>Dataset.map()</code> pentru a aplica funcția noastră <code>get_embeddings()</code> la fiecare rând din corpusul nostru, astfel încât să creăm o nouă coloană <code>embeddings</code> în acest mod:",k,I,$;return a=new j({props:{code:"ZGVmJTIwZ2V0X2VtYmVkZGluZ3ModGV4dF9saXN0KSUzQSUwQSUyMCUyMCUyMCUyMGVuY29kZWRfaW5wdXQlMjAlM0QlMjB0b2tlbml6ZXIoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdGV4dF9saXN0JTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjB0cnVuY2F0aW9uJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIlMEElMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwZW5jb2RlZF9pbnB1dCUyMCUzRCUyMCU3QmslM0ElMjB2JTIwZm9yJTIwayUyQyUyMHYlMjBpbiUyMGVuY29kZWRfaW5wdXQuaXRlbXMoKSU3RCUwQSUyMCUyMCUyMCUyMG1vZGVsX291dHB1dCUyMCUzRCUyMG1vZGVsKCoqZW5jb2RlZF9pbnB1dCklMEElMjAlMjAlMjAlMjByZXR1cm4lMjBjbHNfcG9vbGluZyhtb2RlbF9vdXRwdXQp",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_embeddings</span>(<span class="hljs-params">text_list</span>):
    encoded_input = tokenizer(
        text_list, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>
    )
    encoded_input = {k: v <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> encoded_input.items()}
    model_output = model(**encoded_input)
    <span class="hljs-keyword">return</span> cls_pooling(model_output)`,wrap:!1}}),d=new j({props:{code:"ZW1iZWRkaW5nJTIwJTNEJTIwZ2V0X2VtYmVkZGluZ3MoY29tbWVudHNfZGF0YXNldCU1QiUyMnRleHQlMjIlNUQlNUIwJTVEKSUwQWVtYmVkZGluZy5zaGFwZQ==",highlighted:`embedding = get_embeddings(comments_dataset[<span class="hljs-string">&quot;text&quot;</span>][<span class="hljs-number">0</span>])
embedding.shape`,wrap:!1}}),b=new j({props:{code:"VGVuc29yU2hhcGUoJTVCMSUyQyUyMDc2OCU1RCk=",highlighted:'TensorShape([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])',wrap:!1}}),I=new j({props:{code:"ZW1iZWRkaW5nc19kYXRhc2V0JTIwJTNEJTIwY29tbWVudHNfZGF0YXNldC5tYXAoJTBBJTIwJTIwJTIwJTIwbGFtYmRhJTIweCUzQSUyMCU3QiUyMmVtYmVkZGluZ3MlMjIlM0ElMjBnZXRfZW1iZWRkaW5ncyh4JTVCJTIydGV4dCUyMiU1RCkubnVtcHkoKSU1QjAlNUQlN0QlMEEp",highlighted:`embeddings_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;embeddings&quot;</span>: get_embeddings(x[<span class="hljs-string">&quot;text&quot;</span>]).numpy()[<span class="hljs-number">0</span>]}
)`,wrap:!1}}),{c(){m(a.$$.fragment),o=r(),n=f("p"),n.textContent=h,u=r(),m(d.$$.fragment),G=r(),m(b.$$.fragment),Q=r(),Z=f("p"),Z.innerHTML=R,k=r(),m(I.$$.fragment)},l(i){U(a.$$.fragment,i),o=c(i),n=T(i,"P",{"data-svelte-h":!0}),w(n)!=="svelte-nqgrhi"&&(n.textContent=h),u=c(i),U(d.$$.fragment,i),G=c(i),U(b.$$.fragment,i),Q=c(i),Z=T(i,"P",{"data-svelte-h":!0}),w(Z)!=="svelte-p4v437"&&(Z.innerHTML=R),k=c(i),U(I.$$.fragment,i)},m(i,g){y(a,i,g),s(i,o,g),s(i,n,g),s(i,u,g),y(d,i,g),s(i,G,g),y(b,i,g),s(i,Q,g),s(i,Z,g),s(i,k,g),y(I,i,g),$=!0},i(i){$||(p(a.$$.fragment,i),p(d.$$.fragment,i),p(b.$$.fragment,i),p(I.$$.fragment,i),$=!0)},o(i){M(a.$$.fragment,i),M(d.$$.fragment,i),M(b.$$.fragment,i),M(I.$$.fragment,i),$=!1},d(i){i&&(l(o),l(n),l(u),l(G),l(Q),l(Z),l(k)),J(a,i),J(d,i),J(b,i),J(I,i)}}}function Ms(C){let a,o,n,h="Putem testa funcția oferindui prima intrare de text în corpusul nostru și analizând output shapeul:",u,d,G,b,Q,Z,R="Minunat, am transformat prima intrare din corpusul nostru într-un vector de 768 de dimensiuni! Putem utiliza <code>Dataset.map()</code> pentru a aplica funcția noastră <code>get_embeddings()</code> la fiecare rând din corpusul nostru, astfel încât să creăm o nouă coloană <code>embeddings</code> în acest mod:",k,I,$;return a=new j({props:{code:"ZGVmJTIwZ2V0X2VtYmVkZGluZ3ModGV4dF9saXN0KSUzQSUwQSUyMCUyMCUyMCUyMGVuY29kZWRfaW5wdXQlMjAlM0QlMjB0b2tlbml6ZXIoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdGV4dF9saXN0JTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjB0cnVuY2F0aW9uJTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIlMEElMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwZW5jb2RlZF9pbnB1dCUyMCUzRCUyMCU3QmslM0ElMjB2LnRvKGRldmljZSklMjBmb3IlMjBrJTJDJTIwdiUyMGluJTIwZW5jb2RlZF9pbnB1dC5pdGVtcygpJTdEJTBBJTIwJTIwJTIwJTIwbW9kZWxfb3V0cHV0JTIwJTNEJTIwbW9kZWwoKiplbmNvZGVkX2lucHV0KSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMGNsc19wb29saW5nKG1vZGVsX291dHB1dCk=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_embeddings</span>(<span class="hljs-params">text_list</span>):
    encoded_input = tokenizer(
        text_list, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>
    )
    encoded_input = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> encoded_input.items()}
    model_output = model(**encoded_input)
    <span class="hljs-keyword">return</span> cls_pooling(model_output)`,wrap:!1}}),d=new j({props:{code:"ZW1iZWRkaW5nJTIwJTNEJTIwZ2V0X2VtYmVkZGluZ3MoY29tbWVudHNfZGF0YXNldCU1QiUyMnRleHQlMjIlNUQlNUIwJTVEKSUwQWVtYmVkZGluZy5zaGFwZQ==",highlighted:`embedding = get_embeddings(comments_dataset[<span class="hljs-string">&quot;text&quot;</span>][<span class="hljs-number">0</span>])
embedding.shape`,wrap:!1}}),b=new j({props:{code:"dG9yY2guU2l6ZSglNUIxJTJDJTIwNzY4JTVEKQ==",highlighted:'torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])',wrap:!1}}),I=new j({props:{code:"ZW1iZWRkaW5nc19kYXRhc2V0JTIwJTNEJTIwY29tbWVudHNfZGF0YXNldC5tYXAoJTBBJTIwJTIwJTIwJTIwbGFtYmRhJTIweCUzQSUyMCU3QiUyMmVtYmVkZGluZ3MlMjIlM0ElMjBnZXRfZW1iZWRkaW5ncyh4JTVCJTIydGV4dCUyMiU1RCkuZGV0YWNoKCkuY3B1KCkubnVtcHkoKSU1QjAlNUQlN0QlMEEp",highlighted:`embeddings_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;embeddings&quot;</span>: get_embeddings(x[<span class="hljs-string">&quot;text&quot;</span>]).detach().cpu().numpy()[<span class="hljs-number">0</span>]}
)`,wrap:!1}}),{c(){m(a.$$.fragment),o=r(),n=f("p"),n.textContent=h,u=r(),m(d.$$.fragment),G=r(),m(b.$$.fragment),Q=r(),Z=f("p"),Z.innerHTML=R,k=r(),m(I.$$.fragment)},l(i){U(a.$$.fragment,i),o=c(i),n=T(i,"P",{"data-svelte-h":!0}),w(n)!=="svelte-nqgrhi"&&(n.textContent=h),u=c(i),U(d.$$.fragment,i),G=c(i),U(b.$$.fragment,i),Q=c(i),Z=T(i,"P",{"data-svelte-h":!0}),w(Z)!=="svelte-p4v437"&&(Z.innerHTML=R),k=c(i),U(I.$$.fragment,i)},m(i,g){y(a,i,g),s(i,o,g),s(i,n,g),s(i,u,g),y(d,i,g),s(i,G,g),y(b,i,g),s(i,Q,g),s(i,Z,g),s(i,k,g),y(I,i,g),$=!0},i(i){$||(p(a.$$.fragment,i),p(d.$$.fragment,i),p(b.$$.fragment,i),p(I.$$.fragment,i),$=!0)},o(i){M(a.$$.fragment,i),M(d.$$.fragment,i),M(b.$$.fragment,i),M(I.$$.fragment,i),$=!1},d(i){i&&(l(o),l(n),l(u),l(G),l(Q),l(Z),l(k)),J(a,i),J(d,i),J(b,i),J(I,i)}}}function ps(C){let a,o,n,h;return a=new j({props:{code:"cXVlc3Rpb24lMjAlM0QlMjAlMjJIb3clMjBjYW4lMjBJJTIwbG9hZCUyMGElMjBkYXRhc2V0JTIwb2ZmbGluZSUzRiUyMiUwQXF1ZXN0aW9uX2VtYmVkZGluZyUyMCUzRCUyMGdldF9lbWJlZGRpbmdzKCU1QnF1ZXN0aW9uJTVEKS5udW1weSgpJTBBcXVlc3Rpb25fZW1iZWRkaW5nLnNoYXBl",highlighted:`question = <span class="hljs-string">&quot;How can I load a dataset offline?&quot;</span>
question_embedding = get_embeddings([question]).numpy()
question_embedding.shape`,wrap:!1}}),n=new j({props:{code:"KDElMkMlMjA3Njgp",highlighted:'(<span class="hljs-number">1</span>, <span class="hljs-number">768</span>)',wrap:!1}}),{c(){m(a.$$.fragment),o=r(),m(n.$$.fragment)},l(u){U(a.$$.fragment,u),o=c(u),U(n.$$.fragment,u)},m(u,d){y(a,u,d),s(u,o,d),y(n,u,d),h=!0},i(u){h||(p(a.$$.fragment,u),p(n.$$.fragment,u),h=!0)},o(u){M(a.$$.fragment,u),M(n.$$.fragment,u),h=!1},d(u){u&&l(o),J(a,u),J(n,u)}}}function os(C){let a,o,n,h;return a=new j({props:{code:"cXVlc3Rpb24lMjAlM0QlMjAlMjJIb3clMjBjYW4lMjBJJTIwbG9hZCUyMGElMjBkYXRhc2V0JTIwb2ZmbGluZSUzRiUyMiUwQXF1ZXN0aW9uX2VtYmVkZGluZyUyMCUzRCUyMGdldF9lbWJlZGRpbmdzKCU1QnF1ZXN0aW9uJTVEKS5jcHUoKS5kZXRhY2goKS5udW1weSgpJTBBcXVlc3Rpb25fZW1iZWRkaW5nLnNoYXBl",highlighted:`question = <span class="hljs-string">&quot;How can I load a dataset offline?&quot;</span>
question_embedding = get_embeddings([question]).cpu().detach().numpy()
question_embedding.shape`,wrap:!1}}),n=new j({props:{code:"dG9yY2guU2l6ZSglNUIxJTJDJTIwNzY4JTVEKQ==",highlighted:'torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])',wrap:!1}}),{c(){m(a.$$.fragment),o=r(),m(n.$$.fragment)},l(u){U(a.$$.fragment,u),o=c(u),U(n.$$.fragment,u)},m(u,d){y(a,u,d),s(u,o,d),y(n,u,d),h=!0},i(u){h||(p(a.$$.fragment,u),p(n.$$.fragment,u),h=!0)},o(u){M(a.$$.fragment,u),M(n.$$.fragment,u),h=!1},d(u){u&&l(o),J(a,u),J(n,u)}}}function ds(C){let a,o="✏️ <strong>Încearcă!</strong> Creează propriul tău query și vezi dacp poți găsi un răspuns și să extragi documentele. S-ar putea să trebuiești să crești parametrul <code>k</code> în <code>Dataset.get_nearest_examples()</code> pentru a mări căutarea.";return{c(){a=f("p"),a.innerHTML=o},l(n){a=T(n,"P",{"data-svelte-h":!0}),w(a)!=="svelte-1as03g8"&&(a.innerHTML=o)},m(n,h){s(n,a,h)},p:Hl,d(n){n&&l(a)}}}function ms(C){let a,o,n,h,u,d,G,b,Q,Z,R,k,I='În <a href="/course/chapter5/5">secțiunea 5</a>, am creat un dataset cu issues și comentarii din repositoriul 🤗 Datasets. În această secțiune, vom utiliza aceste informații pentru a construi un motor de căutare care ne poate ajuta să găsim răspunsurile la  cele mai importante cele mai importante întrebări despre bibliotecă!',$,i,g,S,Pe,F,rl='După cum am văzut în <a href="/course/chapter1">Capitolul 1</a>, Transformer-based language models reprezintă fiecare token într-un fragment de text ca un <em>embedding vector</em>. S-a dovedit că se poate face “pool” embeddingurilor individuale pentru a crea o reprezentare vectorială pentru fraze întregi, paragrafe sau (în anumite cazuri) documente. Aceste embeddings pot fi apoi utilizate pentru a găsi documente similare în corpus prin calcularea dot-product similarity (sau a unei alte metrice de similaritate) între fiecare embedding și returnarea documentelor cu cea mai mare suprapunere.',Ke,E,cl="În această secțiune, vom utiliza embeddings pentru a dezvolta un motor de căutare semantică. Aceste motoare de căutare oferă mai multe avantaje față de abordările convenționale bazate pe căutarea de cuvinte cheie într-un query cu documente.",Oe,V,ul='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/semantic-search.svg" alt="Căutare semantică."/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/semantic-search-dark.svg" alt="Căutare semantică."/>',et,A,tt,H,Ml="Prima lucru pe care trebuie să îl facem este să descărcăm datasetul nostru cu GitHub issues, așa că folosim funcția <code>load_dataset()</code> ca de obicei:",lt,q,st,D,at,L,pl="Aici am specificat splitul default <code>train</code> în <code>load_dataset()</code>, astfel încât returnează un <code>Dataset</code> în loc de <code>DatasetDict</code>. Primul lucru care treubuie făcut este să filtrăm pull requesturile, deoarece acestea rareori tind să fie utilizate pentru a răspunde la întrebările utilizatorilor și vor introduce noise în motorul nostru de căutare. Așa cum ar trebuie deja să știți, putem utiliza funcția <code>Dataset.filter()</code> pentru a exclude aceste rânduri din datasetul nostru. În timp ce suntem aici, putem să filtrăm și rândurile fără comentari, deoarece acestea nu oferă niciun răspuns la întrebările utilizatorilor:",nt,P,it,K,rt,O,ol="Putem vedea că există multe coloane în datasetul nostru, majoritatea dintre care nu sunt necesare pentru a construi motorul nostru de căutare. Din perspectiva căutării, cele mai informative coloane sunt <code>title</code>, <code>body</code> și <code>comments</code>, în timp ce <code>html_url</code> ne oferă un link înapoi la problema sursă. Hai să utilizăm funcția <code>Dataset.remove_columns()</code> pentru a elimina restul:",ct,ee,ut,te,Mt,le,dl='Pentru a crea embeddedurile noastre, vom completa fiecare comentariu cu titlul și body-ul problemei, deoarece aceste câmpuri adesea includ informații contextuale utile. Deoarece coloana noastră <code>comments</code> este în prezent o listă de comentarii pentru fiecare issue, trebuie să “explodăm” coloana, astfel încât fiecare rând să fie format dintr-un tuple <code>(html_url, title, body, comment)</code>. În Pandas, putem face acest lucru cu funcția <a href="https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.explode.html" rel="nofollow"><code>DataFrame.explode()</code></a>, care creează un rând nou pentru fiecare element dintr-o coloană asemănătoare cu o listă, în timp ce copiază toate celelalte valori ale coloanelor. Pentru a vedea acest lucru în acțiune, să trecem la formatul pandas <code>DataFrame</code> mai întâi:',pt,se,ot,ae,ml="Dacă inspectăm primul rând din acest <code>DataFrame</code>, putem vedea că există patru comentarii asociate acestei probleme:",dt,ne,mt,ie,Ut,re,Ul="Când facem explode <code>df</code>, ne așteptăm să obținem un rând pentru fiecare dintre aceste comentarii. Haideți să verificăm dacă ăsta e cazul:",yt,ce,Jt,_,yl='<thead><tr style="text-align: right;"><th></th> <th>html_url</th> <th>title</th> <th>comments</th> <th>body</th></tr></thead> <tbody><tr><th>0</th> <td>https://github.com/huggingface/datasets/issues/2787</td> <td>ConnectionError: Couldn&#39;t reach https://raw.githubusercontent.com</td> <td>the bug code locate in ：\\r\\n    if data_args.task_name is not None...</td> <td>Hello,\\r\\nI am trying to run run_glue.py and it gives me this error...</td></tr> <tr><th>1</th> <td>https://github.com/huggingface/datasets/issues/2787</td> <td>ConnectionError: Couldn&#39;t reach https://raw.githubusercontent.com</td> <td>Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of `ConnectionError` coming from the github.com website: https://raw.githubusercontent.com...</td> <td>Hello,\\r\\nI am trying to run run_glue.py and it gives me this error...</td></tr> <tr><th>2</th> <td>https://github.com/huggingface/datasets/issues/2787</td> <td>ConnectionError: Couldn&#39;t reach https://raw.githubusercontent.com</td> <td>cannot connect，even by Web browser，please check that  there is some  problems。</td> <td>Hello,\\r\\nI am trying to run run_glue.py and it gives me this error...</td></tr> <tr><th>3</th> <td>https://github.com/huggingface/datasets/issues/2787</td> <td>ConnectionError: Couldn&#39;t reach https://raw.githubusercontent.com</td> <td>I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem...</td> <td>Hello,\\r\\nI am trying to run run_glue.py and it gives me this error...</td></tr></tbody>',bt,ue,Jl="Great, putem vedea că rândurile au fost reproduse, cu coloanele <code>comments</code> incluzând și comentariile individuale. Acum că am terminat cu Pandas, noi putem să schimăm rapid înapoi la un <code>Dataset</code> înărcând <code>DataFrame</code> în memorie:",ft,Me,Tt,pe,jt,oe,bl="Okay, acest lucru ne-a oferit câteva mii de comentarii cu care să lucrăm!",ht,W,wt,de,fl="Acum că avem un singur comentariu pe rând, să creăm o nouă coloană <code>comments_length</code> care conține numărul de cuvinte din fiecare comentariu:",gt,me,Qt,Ue,Tl="Putem utiliza această nouă coloană pentru a filtra comentariile scurte, care de obicei includ lucruri precum “cc @lewtun” sau “Mulțumesc!” care nu sunt relevante pentru motorul nostru de căutare. Nu există un număr precis care trebuie selectat pentru filtru, dar aproximativ 15 cuvinte pare a fi un bun punct de plecare:",Zt,ye,Gt,Je,Ct,be,jl="După ce am curățat puțin setul nostru de date, putem să concatenăm titlul, descrirea și comentariile problemei împreună într-o nouă coloană <code>text</code>. Ca de obicei, vom scrie o funcție simplă pe care o putem transmite în <code>Dataset.map()</code>:",It,fe,kt,Te,hl="Suntem în final pregătiți să creăm niște embeddings! Haideți să vedem cum facem acest lucru.",$t,je,Rt,he,wl='Am văzut în <a href="/course/chapter2">Capitolul 2</a> că putem obține token embeddings prin utilizarea clasei <code>AutoModel</code>. Tot ce trebuie să facem este să alegem un checkpoint potrivit pentru a încărca modelul. Din fericire, există o bibliotecă numită <code>sentence-transformers</code> care se ocupă de crearea embeddingurilor. Așa cum se descrie în <a href="https://www.sbert.net/examples/applications/semantic-search/README.html#symmetric-vs-asymmetric-semantic-search" rel="nofollow">documentația</a> bibliotecii, cazul nostru este un exemplu de <em>căutare semantică asimetrică</em> deoarece avem o întrebare scurtă al cărei răspuns ne-ar plăcea să îl găsim într-un document mai lung, precum un comentariu la un issue. [Tabelul] (<a href="https://www.sbert.net/docs/pretrained_models.html#model-overview" rel="nofollow">https://www.sbert.net/docs/pretrained_models.html#model-overview</a>) util de prezentare a modelului din documentație indică faptul că checkpointul <code>multi-qa-mpnet-base-dot-v1</code> are cea mai bună performanță pentru căutarea semantică, așa că îl vom folosi acesta pentru aplicația noastră. De asemenea, vom încărca tokenizer-ul folosind același checkpoint:',_t,N,x,Ee,we,gl="După cum am menționat anterior, dorim să reprezentăm fiecare intrare din corpusul nostru de GitHub issues sub forma unui vector, astfel încât avem nevoie să “agregăm” sau să facem media la tokem embeddings într-un anumit mod. O abordare populară este de a efectua <em>CLS pooling</em> pe outputurile modelului nostru, unde pur și simplu colectăm ultimul stadiu ascuns pentru tokenul special <code>[CLS]</code>. Următoarea funcție face acest lucru pentru noi:",Nt,ge,xt,Qe,Ql="În continuare, vom crea o funcție ajutătoare care va tokeniza o listă de documente, va plasa tensorii pe GPU, îi va alimenta în model și, în final, va aplica CLS pooling la outputuri:",Bt,B,z,Ae,Ze,Zl="Observăm că am transformat embeddingurile în matrice NumPy — acest lucru este necesar deoarece biblioteca 🤗 Datasets cere acest format atunci când încercăm să indexăm cu FAISS, ceea ce vom face în continuare.",zt,Ge,vt,Ce,Gl='Acum că avem un dataset de embeddings, avem nevoie de o modalitate de a căuta printre ele. Pentru a face acest lucru, vom utiliza o structură de date specială din 🤗 Datasets, numită <em>FAISS index</em>. <a href="https://faiss.ai/" rel="nofollow">FAISS</a> (prescurtat de la Facebook AI Similarity Search) este o bibliotecă care oferă algoritmi eficienți pentru căutarea rapidă și clusteringul al embedding vectors.',Xt,Ie,Cl="Ideea de bază din spatele FAISS este crearea unei structuri de date speciale numite <em>index</em>, care permite găsirea embeddingurilor similare cu un input embedding. Crearea unui index FAISS în 🤗 Datasets este simplu — utilizăm funcția <code>Dataset.add_faiss_index()</code> și specificăm care coloană a datasetului nostru dorim să indexăm:",Vt,ke,Wt,$e,Il="Acum putem efectua queries pe acest index realizând o căutare a celor mai apropiați vecini cu funcția <code>Dataset.get_nearest_examples()</code>. Haideți să testăm acest lucru prin încorporarea unei întrebări astfel:",Yt,v,X,He,Re,kl="La fel ca și în cazul documentelor, acum avem un vector de 768 de dimensiuni care reprezintă query-ul, pe care îl putem compara cu întregul corpus pentru a găsi embeddingurile cele mai similare:",St,_e,Ft,Ne,$l="Funcția <code>Dataset.get_nearest_examples()</code> returnează un tuple cu scorurile care clasifică suprapunerea dintre query și document, și un set corespunzător de sampleuri (în acest caz, cele 5 match-uri). Haideți să colectăm acestea într-un <code>pandas.DataFrame</code> pentru a le putea sorta cu ușurință:",Et,xe,At,Be,Rl="Acum putem itera peste primele rânduri pentru a veadea cât de bine un query se potrivește cu comentariile disponibile:",Ht,ze,qt,ve,Dt,Xe,_l="Nu-i rău! A doua încercare se pare că se potrivește cu query-ul!",Lt,Y,Pt,Ve,Kt,qe,Ot;u=new ls({props:{fw:C[0]}}),G=new Le({props:{title:"Căutare semantică cu FAISS",local:"semantic-search-with-faiss",headingTag:"h1"}});const Nl=[ns,as],We=[];function xl(e,t){return e[0]==="pt"?0:1}Q=xl(C),Z=We[Q]=Nl[Q](C),i=new ts({props:{id:"OATCgQtNX2o"}}),S=new Le({props:{title:"Utilizarea embeddings pentru căutare semantică",local:"using-embeddings-for-semantic-search",headingTag:"h2"}}),A=new Le({props:{title:"Încărcarea și pregătirea datasetului",local:"loading-and-preparing-the-dataset",headingTag:"h2"}}),q=new j({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBaXNzdWVzX2RhdGFzZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIybGV3dHVuJTJGZ2l0aHViLWlzc3VlcyUyMiUyQyUyMHNwbGl0JTNEJTIydHJhaW4lMjIpJTBBaXNzdWVzX2RhdGFzZXQ=",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

issues_dataset = load_dataset(<span class="hljs-string">&quot;lewtun/github-issues&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
issues_dataset`,wrap:!1}}),D=new j({props:{code:"RGF0YXNldCglN0IlMEElMjAlMjAlMjAlMjBmZWF0dXJlcyUzQSUyMCU1Qid1cmwnJTJDJTIwJ3JlcG9zaXRvcnlfdXJsJyUyQyUyMCdsYWJlbHNfdXJsJyUyQyUyMCdjb21tZW50c191cmwnJTJDJTIwJ2V2ZW50c191cmwnJTJDJTIwJ2h0bWxfdXJsJyUyQyUyMCdpZCclMkMlMjAnbm9kZV9pZCclMkMlMjAnbnVtYmVyJyUyQyUyMCd0aXRsZSclMkMlMjAndXNlciclMkMlMjAnbGFiZWxzJyUyQyUyMCdzdGF0ZSclMkMlMjAnbG9ja2VkJyUyQyUyMCdhc3NpZ25lZSclMkMlMjAnYXNzaWduZWVzJyUyQyUyMCdtaWxlc3RvbmUnJTJDJTIwJ2NvbW1lbnRzJyUyQyUyMCdjcmVhdGVkX2F0JyUyQyUyMCd1cGRhdGVkX2F0JyUyQyUyMCdjbG9zZWRfYXQnJTJDJTIwJ2F1dGhvcl9hc3NvY2lhdGlvbiclMkMlMjAnYWN0aXZlX2xvY2tfcmVhc29uJyUyQyUyMCdwdWxsX3JlcXVlc3QnJTJDJTIwJ2JvZHknJTJDJTIwJ3BlcmZvcm1lZF92aWFfZ2l0aHViX2FwcCclMkMlMjAnaXNfcHVsbF9yZXF1ZXN0JyU1RCUyQyUwQSUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwMjg1NSUwQSU3RCk=",highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>, <span class="hljs-string">&#x27;is_pull_request&#x27;</span>],
    num_rows: <span class="hljs-number">2855</span>
})`,wrap:!1}}),P=new j({props:{code:"aXNzdWVzX2RhdGFzZXQlMjAlM0QlMjBpc3N1ZXNfZGF0YXNldC5maWx0ZXIoJTBBJTIwJTIwJTIwJTIwbGFtYmRhJTIweCUzQSUyMCh4JTVCJTIyaXNfcHVsbF9yZXF1ZXN0JTIyJTVEJTIwJTNEJTNEJTIwRmFsc2UlMjBhbmQlMjBsZW4oeCU1QiUyMmNvbW1lbnRzJTIyJTVEKSUyMCUzRSUyMDApJTBBKSUwQWlzc3Vlc19kYXRhc2V0",highlighted:`issues_dataset = issues_dataset.<span class="hljs-built_in">filter</span>(
    <span class="hljs-keyword">lambda</span> x: (x[<span class="hljs-string">&quot;is_pull_request&quot;</span>] == <span class="hljs-literal">False</span> <span class="hljs-keyword">and</span> <span class="hljs-built_in">len</span>(x[<span class="hljs-string">&quot;comments&quot;</span>]) &gt; <span class="hljs-number">0</span>)
)
issues_dataset`,wrap:!1}}),K=new j({props:{code:"RGF0YXNldCglN0IlMEElMjAlMjAlMjAlMjBjYXJhY3RlcmlzdGljaSUzQSUyMCU1Qid1cmwnJTJDJTIwJ3JlcG9zaXRvcnlfdXJsJyUyQyUyMCdsYWJlbHNfdXJsJyUyQyUyMCdjb21tZW50c191cmwnJTJDJTIwJ2V2ZW50c191cmwnJTJDJTIwJ2h0bWxfdXJsJyUyQyUyMCdpZCclMkMlMjAnbm9kZV9pZCclMkMlMjAnbnVtYmVyJyUyQyUyMCd0aXRsZSclMkMlMjAndXNlciclMkMlMjAnbGFiZWxzJyUyQyUyMCdzdGF0ZSclMkMlMjAnbG9ja2VkJyUyQyUyMCdhc3NpZ25lZSclMkMlMjAnYXNzaWduZWVzJyUyQyUyMCdtaWxlc3RvbmUnJTJDJTIwJ2NvbW1lbnRzJyUyQyUyMCdjcmVhdGVkX2F0JyUyQyUyMCd1cGRhdGVkX2F0JyUyQyUyMCdjbG9zZWRfYXQnJTJDJTIwJ2F1dGhvcl9hc3NvY2lhdGlvbiclMkMlMjAnYWN0aXZlX2xvY2tfcmVhc29uJyUyQyUyMCdwdWxsX3JlcXVlc3QnJTJDJTIwJ2JvZHknJTJDJTIwJ3BlcmZvcm1lZF92aWFfZ2l0aHViX2FwcCclMkMlMjAnaXNfcHVsbF9yZXF1ZXN0JyU1RCUyQyUwQSUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwNzcxJTBBJTdEKQ==",highlighted:`Dataset({
    caracteristici: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>, <span class="hljs-string">&#x27;is_pull_request&#x27;</span>],
    num_rows: <span class="hljs-number">771</span>
})`,wrap:!1}}),ee=new j({props:{code:"Y29sdW1ucyUyMCUzRCUyMGlzc3Vlc19kYXRhc2V0LmNvbHVtbl9uYW1lcyUwQWNvbHVtbnNfdG9fa2VlcCUyMCUzRCUyMCU1QiUyMnRpdGxlJTIyJTJDJTIwJTIyYm9keSUyMiUyQyUyMCUyMmh0bWxfdXJsJTIyJTJDJTIwJTIyY29tbWVudHMlMjIlNUQlMEFjb2x1bW5zX3RvX3JlbW92ZSUyMCUzRCUyMHNldChjb2x1bW5zX3RvX2tlZXApLnN5bW1ldHJpY19kaWZmZXJlbmNlKGNvbHVtbnMpJTBBaXNzdWVzX2RhdGFzZXQlMjAlM0QlMjBpc3N1ZXNfZGF0YXNldC5yZW1vdmVfY29sdW1ucyhjb2x1bW5zX3RvX3JlbW92ZSklMEFpc3N1ZXNfZGF0YXNldA==",highlighted:`columns = issues_dataset.column_names
columns_to_keep = [<span class="hljs-string">&quot;title&quot;</span>, <span class="hljs-string">&quot;body&quot;</span>, <span class="hljs-string">&quot;html_url&quot;</span>, <span class="hljs-string">&quot;comments&quot;</span>]
columns_to_remove = <span class="hljs-built_in">set</span>(columns_to_keep).symmetric_difference(columns)
issues_dataset = issues_dataset.remove_columns(columns_to_remove)
issues_dataset`,wrap:!1}}),te=new j({props:{code:"RGF0YXNldCglN0IlMEElMjAlMjAlMjAlMjBmZWF0dXJlcyUzQSUyMCU1QidodG1sX3VybCclMkMlMjAndGl0bGUnJTJDJTIwJ2NvbW1lbnRzJyUyQyUyMCdib2R5JyU1RCUyQyUwQSUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwNzcxJTBBJTdEKQ==",highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>],
    num_rows: <span class="hljs-number">771</span>
})`,wrap:!1}}),se=new j({props:{code:"aXNzdWVzX2RhdGFzZXQuc2V0X2Zvcm1hdCglMjJwYW5kYXMlMjIpJTBBZGYlMjAlM0QlMjBpc3N1ZXNfZGF0YXNldCU1QiUzQSU1RA==",highlighted:`issues_dataset.set_format(<span class="hljs-string">&quot;pandas&quot;</span>)
df = issues_dataset[:]`,wrap:!1}}),ne=new j({props:{code:"ZGYlNUIlMjJjb21tZW50cyUyMiU1RCU1QjAlNUQudG9saXN0KCk=",highlighted:'df[<span class="hljs-string">&quot;comments&quot;</span>][<span class="hljs-number">0</span>].tolist()',wrap:!1}}),ie=new j({props:{code:"JTVCJ3RoZSUyMGJ1ZyUyMGNvZGUlMjBsb2NhdGUlMjBpbiUyMCVFRiVCQyU5QSU1Q3IlNUNuJTIwJTIwJTIwJTIwaWYlMjBkYXRhX2FyZ3MudGFza19uYW1lJTIwaXMlMjBub3QlMjBOb25lJTNBJTVDciU1Q24lMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBEb3dubG9hZGluZyUyMGFuZCUyMGxvYWRpbmclMjBhJTIwZGF0YXNldCUyMGZyb20lMjB0aGUlMjBodWIuJTVDciU1Q24lMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBkYXRhc2V0cyUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJnbHVlJTIyJTJDJTIwZGF0YV9hcmdzLnRhc2tfbmFtZSUyQyUyMGNhY2hlX2RpciUzRG1vZGVsX2FyZ3MuY2FjaGVfZGlyKSclMkMlMEElMjAnSGklMjAlNDBqaW5lYyUyQyU1Q3IlNUNuJTVDciU1Q25Gcm9tJTIwdGltZSUyMHRvJTIwdGltZSUyMHdlJTIwZ2V0JTIwdGhpcyUyMGtpbmQlMjBvZiUyMCU2MENvbm5lY3Rpb25FcnJvciU2MCUyMGNvbWluZyUyMGZyb20lMjB0aGUlMjBnaXRodWIuY29tJTIwd2Vic2l0ZSUzQSUyMGh0dHBzJTNBJTJGJTJGcmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSU1Q3IlNUNuJTVDciU1Q25Ob3JtYWxseSUyQyUyMGl0JTIwc2hvdWxkJTIwd29yayUyMGlmJTIweW91JTIwd2FpdCUyMGElMjBsaXR0bGUlMjBhbmQlMjB0aGVuJTIwcmV0cnkuJTVDciU1Q24lNUNyJTVDbkNvdWxkJTIweW91JTIwcGxlYXNlJTIwY29uZmlybSUyMGlmJTIwdGhlJTIwcHJvYmxlbSUyMHBlcnNpc3RzJTNGJyUyQyUwQSUyMCdjYW5ub3QlMjBjb25uZWN0JUVGJUJDJThDZXZlbiUyMGJ5JTIwV2ViJTIwYnJvd3NlciVFRiVCQyU4Q3BsZWFzZSUyMGNoZWNrJTIwdGhhdCUyMCUyMHRoZXJlJTIwaXMlMjBzb21lJTIwJTIwcHJvYmxlbXMlRTMlODAlODInJTJDJTBBJTIwJ0klMjBjYW4lMjBhY2Nlc3MlMjBodHRwcyUzQSUyRiUyRnJhdy5naXRodWJ1c2VyY29udGVudC5jb20lMkZodWdnaW5nZmFjZSUyRmRhdGFzZXRzJTJGMS43LjAlMkZkYXRhc2V0cyUyRmdsdWUlMkZnbHVlLnB5JTIwd2l0aG91dCUyMHByb2JsZW0uLi4nJTVE",highlighted:`[<span class="hljs-string">&#x27;the bug code locate in ：\\r\\n    if data_args.task_name is not None:\\r\\n        # Downloading and loading a dataset from the hub.\\r\\n        datasets = load_dataset(&quot;glue&quot;, data_args.task_name, cache_dir=model_args.cache_dir)&#x27;</span>,
 <span class="hljs-string">&#x27;Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of \`ConnectionError\` coming from the github.com website: https://raw.githubusercontent.com\\r\\n\\r\\nNormally, it should work if you wait a little and then retry.\\r\\n\\r\\nCould you please confirm if the problem persists?&#x27;</span>,
 <span class="hljs-string">&#x27;cannot connect，even by Web browser，please check that  there is some  problems。&#x27;</span>,
 <span class="hljs-string">&#x27;I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem...&#x27;</span>]`,wrap:!1}}),ce=new j({props:{code:"Y29tbWVudHNfZGYlMjAlM0QlMjBkZi5leHBsb2RlKCUyMmNvbW1lbnRzJTIyJTJDJTIwaWdub3JlX2luZGV4JTNEVHJ1ZSklMEFjb21tZW50c19kZi5oZWFkKDQp",highlighted:`comments_df = df.explode(<span class="hljs-string">&quot;comments&quot;</span>, ignore_index=<span class="hljs-literal">True</span>)
comments_df.head(<span class="hljs-number">4</span>)`,wrap:!1}}),Me=new j({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQSUwQWNvbW1lbnRzX2RhdGFzZXQlMjAlM0QlMjBEYXRhc2V0LmZyb21fcGFuZGFzKGNvbW1lbnRzX2RmKSUwQWNvbW1lbnRzX2RhdGFzZXQ=",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset

comments_dataset = Dataset.from_pandas(comments_df)
comments_dataset`,wrap:!1}}),pe=new j({props:{code:"RGF0YXNldCglN0IlMEElMjAlMjAlMjAlMjBmZWF0dXJlcyUzQSUyMCU1QidodG1sX3VybCclMkMlMjAndGl0bGUnJTJDJTIwJ2NvbW1lbnRzJyUyQyUyMCdib2R5JyU1RCUyQyUwQSUyMCUyMCUyMCUyMG51bV9yb3dzJTNBJTIwMjg0MiUwQSU3RCk=",highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>],
    num_rows: <span class="hljs-number">2842</span>
})`,wrap:!1}}),W=new Al({props:{$$slots:{default:[is]},$$scope:{ctx:C}}}),me=new j({props:{code:"Y29tbWVudHNfZGF0YXNldCUyMCUzRCUyMGNvbW1lbnRzX2RhdGFzZXQubWFwKCUwQSUyMCUyMCUyMCUyMGxhbWJkYSUyMHglM0ElMjAlN0IlMjJjb21tZW50X2xlbmd0aCUyMiUzQSUyMGxlbih4JTVCJTIyY29tbWVudHMlMjIlNUQuc3BsaXQoKSklN0QlMEEp",highlighted:`comments_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;comment_length&quot;</span>: <span class="hljs-built_in">len</span>(x[<span class="hljs-string">&quot;comments&quot;</span>].split())}
)`,wrap:!1}}),ye=new j({props:{code:"Y29tbWVudHNfZGF0YXNldCUyMCUzRCUyMGNvbW1lbnRzX2RhdGFzZXQuZmlsdGVyKGxhbWJkYSUyMHglM0ElMjB4JTVCJTIyY29tbWVudF9sZW5ndGglMjIlNUQlMjAlM0UlMjAxNSklMEFjb21tZW50c19kYXRhc2V0",highlighted:`comments_dataset = comments_dataset.<span class="hljs-built_in">filter</span>(<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-string">&quot;comment_length&quot;</span>] &gt; <span class="hljs-number">15</span>)
comments_dataset`,wrap:!1}}),Je=new j({props:{code:"RGF0YXNldCglN0IlMEElMjAlMjAlMjAlMjBmZWF0dXJlcyUzQSUyMCU1QidodG1sX3VybCclMkMlMjAndGl0bGUnJTJDJTIwJ2NvbW1lbnRzJyUyQyUyMCdib2R5JyUyQyUyMCdjb21tZW50X2xlbmd0aCclNUQlMkMlMEElMjAlMjAlMjAlMjBudW1fcm93cyUzQSUyMDIwOTglMEElN0Qp",highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;comment_length&#x27;</span>],
    num_rows: <span class="hljs-number">2098</span>
})`,wrap:!1}}),fe=new j({props:{code:"ZGVmJTIwY29uY2F0ZW5hdGVfdGV4dChleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0ZXh0JTIyJTNBJTIwZXhhbXBsZXMlNUIlMjJ0aXRsZSUyMiU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyQiUyMCUyMiUyMCU1Q24lMjAlMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMkIlMjBleGFtcGxlcyU1QiUyMmJvZHklMjIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMkIlMjAlMjIlMjAlNUNuJTIwJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTJCJTIwZXhhbXBsZXMlNUIlMjJjb21tZW50cyUyMiU1RCUwQSUyMCUyMCUyMCUyMCU3RCUwQSUwQSUwQWNvbW1lbnRzX2RhdGFzZXQlMjAlM0QlMjBjb21tZW50c19kYXRhc2V0Lm1hcChjb25jYXRlbmF0ZV90ZXh0KQ==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">concatenate_text</span>(<span class="hljs-params">examples</span>):
    <span class="hljs-keyword">return</span> {
        <span class="hljs-string">&quot;text&quot;</span>: examples[<span class="hljs-string">&quot;title&quot;</span>]
        + <span class="hljs-string">&quot; \\n &quot;</span>
        + examples[<span class="hljs-string">&quot;body&quot;</span>]
        + <span class="hljs-string">&quot; \\n &quot;</span>
        + examples[<span class="hljs-string">&quot;comments&quot;</span>]
    }


comments_dataset = comments_dataset.<span class="hljs-built_in">map</span>(concatenate_text)`,wrap:!1}}),je=new Le({props:{title:"Crearea embeddings-urilor de text",local:"creating-text-embeddings",headingTag:"h2"}});const Bl=[cs,rs],Ye=[];function zl(e,t){return e[0]==="pt"?0:1}N=zl(C),x=Ye[N]=Bl[N](C),ge=new j({props:{code:"ZGVmJTIwY2xzX3Bvb2xpbmcobW9kZWxfb3V0cHV0KSUzQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMG1vZGVsX291dHB1dC5sYXN0X2hpZGRlbl9zdGF0ZSU1QiUzQSUyQyUyMDAlNUQ=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">cls_pooling</span>(<span class="hljs-params">model_output</span>):
    <span class="hljs-keyword">return</span> model_output.last_hidden_state[:, <span class="hljs-number">0</span>]`,wrap:!1}});const vl=[Ms,us],Se=[];function Xl(e,t){return e[0]==="pt"?0:1}B=Xl(C),z=Se[B]=vl[B](C),Ge=new Le({props:{title:"Utilizarea FAISS pentru căutare de similaritate eficientă",local:"using-faiss-for-efficient-similarity-search",headingTag:"h2"}}),ke=new j({props:{code:"ZW1iZWRkaW5nc19kYXRhc2V0LmFkZF9mYWlzc19pbmRleChjb2x1bW4lM0QlMjJlbWJlZGRpbmdzJTIyKQ==",highlighted:'embeddings_dataset.add_faiss_index(column=<span class="hljs-string">&quot;embeddings&quot;</span>)',wrap:!1}});const Vl=[os,ps],Fe=[];function Wl(e,t){return e[0]==="pt"?0:1}return v=Wl(C),X=Fe[v]=Vl[v](C),_e=new j({props:{code:"c2NvcmVzJTJDJTIwc2FtcGxlcyUyMCUzRCUyMGVtYmVkZGluZ3NfZGF0YXNldC5nZXRfbmVhcmVzdF9leGFtcGxlcyglMEElMjAlMjAlMjAlMjAlMjJlbWJlZGRpbmdzJTIyJTJDJTIwcXVlc3Rpb25fZW1iZWRkaW5nJTJDJTIwayUzRDUlMEEp",highlighted:`scores, samples = embeddings_dataset.get_nearest_examples(
    <span class="hljs-string">&quot;embeddings&quot;</span>, question_embedding, k=<span class="hljs-number">5</span>
)`,wrap:!1}}),xe=new j({props:{code:"aW1wb3J0JTIwcGFuZGFzJTIwYXMlMjBwZCUwQSUwQXNhbXBsZXNfZGYlMjAlM0QlMjBwZC5EYXRhRnJhbWUuZnJvbV9kaWN0KHNhbXBsZXMpJTBBc2FtcGxlc19kZiU1QiUyMnNjb3JlcyUyMiU1RCUyMCUzRCUyMHNjb3JlcyUwQXNhbXBsZXNfZGYuc29ydF92YWx1ZXMoJTIyc2NvcmVzJTIyJTJDJTIwYXNjZW5kaW5nJTNERmFsc2UlMkMlMjBpbnBsYWNlJTNEVHJ1ZSk=",highlighted:`<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd

samples_df = pd.DataFrame.from_dict(samples)
samples_df[<span class="hljs-string">&quot;scores&quot;</span>] = scores
samples_df.sort_values(<span class="hljs-string">&quot;scores&quot;</span>, ascending=<span class="hljs-literal">False</span>, inplace=<span class="hljs-literal">True</span>)`,wrap:!1}}),ze=new j({props:{code:"Zm9yJTIwXyUyQyUyMHJvdyUyMGluJTIwc2FtcGxlc19kZi5pdGVycm93cygpJTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMkNPTU1FTlQlM0ElMjAlN0Jyb3cuY29tbWVudHMlN0QlMjIpJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMlNDT1JFJTNBJTIwJTdCcm93LnNjb3JlcyU3RCUyMiklMEElMjAlMjAlMjAlMjBwcmludChmJTIyVElUTEUlM0ElMjAlN0Jyb3cudGl0bGUlN0QlMjIpJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMlVSTCUzQSUyMCU3QnJvdy5odG1sX3VybCU3RCUyMiklMEElMjAlMjAlMjAlMjBwcmludCglMjIlM0QlMjIlMjAqJTIwNTApJTBBJTIwJTIwJTIwJTIwcHJpbnQoKQ==",highlighted:`<span class="hljs-keyword">for</span> _, row <span class="hljs-keyword">in</span> samples_df.iterrows():
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;COMMENT: <span class="hljs-subst">{row.comments}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;SCORE: <span class="hljs-subst">{row.scores}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;TITLE: <span class="hljs-subst">{row.title}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;URL: <span class="hljs-subst">{row.html_url}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;=&quot;</span> * <span class="hljs-number">50</span>)
    <span class="hljs-built_in">print</span>()`,wrap:!1}}),ve=new j({props:{code:" ",highlighted:`<span class="hljs-string">&quot;&quot;&quot;
COMMENT: Requiring online connection is a deal breaker in some cases unfortunately so it&#x27;d be great if offline mode is added similar to how \`transformers\` loads models offline fine.

@mandubian&#x27;s second bullet point suggests that there&#x27;s a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?
SCORE: 25.505046844482422
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: The local dataset builders (csv, text , json and pandas) are now part of the \`datasets\` package since #1726 :)
You can now use them offline
\\\`\\\`\\\`python
datasets = load_dataset(&quot;text&quot;, data_files=data_files)
\\\`\\\`\\\`

We&#x27;ll do a new release soon
SCORE: 24.555509567260742
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: I opened a PR that allows to reload modules that have already been loaded once even if there&#x27;s no internet.

Let me know if you know other ways that can make the offline mode experience better. I&#x27;d be happy to add them :)

I already note the &quot;freeze&quot; modules option, to prevent local modules updates. It would be a cool feature.

----------

&gt; @mandubian&#x27;s second bullet point suggests that there&#x27;s a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?

Indeed \`load_dataset\` allows to load remote dataset script (squad, glue, etc.) but also you own local ones.
For example if you have a dataset script at \`./my_dataset/my_dataset.py\` then you can do
\\\`\\\`\\\`python
load_dataset(&quot;./my_dataset&quot;)
\\\`\\\`\\\`
and the dataset script will generate your dataset once and for all.

----------

About I&#x27;m looking into having \`csv\`, \`json\`, \`text\`, \`pandas\` dataset builders already included in the \`datasets\` package, so that they are available offline by default, as opposed to the other datasets that require the script to be downloaded.
cf #1724
SCORE: 24.14896583557129
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: &gt; here is my way to load a dataset offline, but it **requires** an online machine
&gt;
&gt; 1. (online machine)
&gt;
&gt; \`\`\`
&gt;
&gt; import datasets
&gt;
&gt; data = datasets.load_dataset(...)
&gt;
&gt; data.save_to_disk(/YOUR/DATASET/DIR)
&gt;
&gt; \`\`\`
&gt;
&gt; 2. copy the dir from online to the offline machine
&gt;
&gt; 3. (offline machine)
&gt;
&gt; \`\`\`
&gt;
&gt; import datasets
&gt;
&gt; data = datasets.load_from_disk(/SAVED/DATA/DIR)
&gt;
&gt; \`\`\`
&gt;
&gt;
&gt;
&gt; HTH.


SCORE: 22.893993377685547
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: here is my way to load a dataset offline, but it **requires** an online machine
1. (online machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_dataset(...)
data.save_to_disk(/YOUR/DATASET/DIR)
\\\`\\\`\\\`
2. copy the dir from online to the offline machine
3. (offline machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_from_disk(/SAVED/DATA/DIR)
\\\`\\\`\\\`

HTH.
SCORE: 22.406635284423828
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================
&quot;&quot;&quot;</span>`,wrap:!1}}),Y=new Al({props:{$$slots:{default:[ds]},$$scope:{ctx:C}}}),Ve=new ss({props:{source:"https://github.com/huggingface/course/blob/main/chapters/rum/chapter5/6.mdx"}}),{c(){a=f("meta"),o=r(),n=f("p"),h=r(),m(u.$$.fragment),d=r(),m(G.$$.fragment),b=r(),Z.c(),R=r(),k=f("p"),k.innerHTML=I,$=r(),m(i.$$.fragment),g=r(),m(S.$$.fragment),Pe=r(),F=f("p"),F.innerHTML=rl,Ke=r(),E=f("p"),E.textContent=cl,Oe=r(),V=f("div"),V.innerHTML=ul,et=r(),m(A.$$.fragment),tt=r(),H=f("p"),H.innerHTML=Ml,lt=r(),m(q.$$.fragment),st=r(),m(D.$$.fragment),at=r(),L=f("p"),L.innerHTML=pl,nt=r(),m(P.$$.fragment),it=r(),m(K.$$.fragment),rt=r(),O=f("p"),O.innerHTML=ol,ct=r(),m(ee.$$.fragment),ut=r(),m(te.$$.fragment),Mt=r(),le=f("p"),le.innerHTML=dl,pt=r(),m(se.$$.fragment),ot=r(),ae=f("p"),ae.innerHTML=ml,dt=r(),m(ne.$$.fragment),mt=r(),m(ie.$$.fragment),Ut=r(),re=f("p"),re.innerHTML=Ul,yt=r(),m(ce.$$.fragment),Jt=r(),_=f("table"),_.innerHTML=yl,bt=r(),ue=f("p"),ue.innerHTML=Jl,ft=r(),m(Me.$$.fragment),Tt=r(),m(pe.$$.fragment),jt=r(),oe=f("p"),oe.textContent=bl,ht=r(),m(W.$$.fragment),wt=r(),de=f("p"),de.innerHTML=fl,gt=r(),m(me.$$.fragment),Qt=r(),Ue=f("p"),Ue.textContent=Tl,Zt=r(),m(ye.$$.fragment),Gt=r(),m(Je.$$.fragment),Ct=r(),be=f("p"),be.innerHTML=jl,It=r(),m(fe.$$.fragment),kt=r(),Te=f("p"),Te.textContent=hl,$t=r(),m(je.$$.fragment),Rt=r(),he=f("p"),he.innerHTML=wl,_t=r(),x.c(),Ee=r(),we=f("p"),we.innerHTML=gl,Nt=r(),m(ge.$$.fragment),xt=r(),Qe=f("p"),Qe.textContent=Ql,Bt=r(),z.c(),Ae=r(),Ze=f("p"),Ze.textContent=Zl,zt=r(),m(Ge.$$.fragment),vt=r(),Ce=f("p"),Ce.innerHTML=Gl,Xt=r(),Ie=f("p"),Ie.innerHTML=Cl,Vt=r(),m(ke.$$.fragment),Wt=r(),$e=f("p"),$e.innerHTML=Il,Yt=r(),X.c(),He=r(),Re=f("p"),Re.textContent=kl,St=r(),m(_e.$$.fragment),Ft=r(),Ne=f("p"),Ne.innerHTML=$l,Et=r(),m(xe.$$.fragment),At=r(),Be=f("p"),Be.textContent=Rl,Ht=r(),m(ze.$$.fragment),qt=r(),m(ve.$$.fragment),Dt=r(),Xe=f("p"),Xe.textContent=_l,Lt=r(),m(Y.$$.fragment),Pt=r(),m(Ve.$$.fragment),Kt=r(),qe=f("p"),this.h()},l(e){const t=Ol("svelte-u9bgzb",document.head);a=T(t,"META",{name:!0,content:!0}),t.forEach(l),o=c(e),n=T(e,"P",{}),El(n).forEach(l),h=c(e),U(u.$$.fragment,e),d=c(e),U(G.$$.fragment,e),b=c(e),Z.l(e),R=c(e),k=T(e,"P",{"data-svelte-h":!0}),w(k)!=="svelte-ky680v"&&(k.innerHTML=I),$=c(e),U(i.$$.fragment,e),g=c(e),U(S.$$.fragment,e),Pe=c(e),F=T(e,"P",{"data-svelte-h":!0}),w(F)!=="svelte-126wabi"&&(F.innerHTML=rl),Ke=c(e),E=T(e,"P",{"data-svelte-h":!0}),w(E)!=="svelte-5jfpph"&&(E.textContent=cl),Oe=c(e),V=T(e,"DIV",{class:!0,"data-svelte-h":!0}),w(V)!=="svelte-ombmkb"&&(V.innerHTML=ul),et=c(e),U(A.$$.fragment,e),tt=c(e),H=T(e,"P",{"data-svelte-h":!0}),w(H)!=="svelte-dpm7gd"&&(H.innerHTML=Ml),lt=c(e),U(q.$$.fragment,e),st=c(e),U(D.$$.fragment,e),at=c(e),L=T(e,"P",{"data-svelte-h":!0}),w(L)!=="svelte-1j9ksj"&&(L.innerHTML=pl),nt=c(e),U(P.$$.fragment,e),it=c(e),U(K.$$.fragment,e),rt=c(e),O=T(e,"P",{"data-svelte-h":!0}),w(O)!=="svelte-lg3h7h"&&(O.innerHTML=ol),ct=c(e),U(ee.$$.fragment,e),ut=c(e),U(te.$$.fragment,e),Mt=c(e),le=T(e,"P",{"data-svelte-h":!0}),w(le)!=="svelte-1f2m6qj"&&(le.innerHTML=dl),pt=c(e),U(se.$$.fragment,e),ot=c(e),ae=T(e,"P",{"data-svelte-h":!0}),w(ae)!=="svelte-156gjqh"&&(ae.innerHTML=ml),dt=c(e),U(ne.$$.fragment,e),mt=c(e),U(ie.$$.fragment,e),Ut=c(e),re=T(e,"P",{"data-svelte-h":!0}),w(re)!=="svelte-1dcvwa7"&&(re.innerHTML=Ul),yt=c(e),U(ce.$$.fragment,e),Jt=c(e),_=T(e,"TABLE",{border:!0,class:!0,style:!0,"data-svelte-h":!0}),w(_)!=="svelte-1g5whzd"&&(_.innerHTML=yl),bt=c(e),ue=T(e,"P",{"data-svelte-h":!0}),w(ue)!=="svelte-kh0qqf"&&(ue.innerHTML=Jl),ft=c(e),U(Me.$$.fragment,e),Tt=c(e),U(pe.$$.fragment,e),jt=c(e),oe=T(e,"P",{"data-svelte-h":!0}),w(oe)!=="svelte-v5a6j7"&&(oe.textContent=bl),ht=c(e),U(W.$$.fragment,e),wt=c(e),de=T(e,"P",{"data-svelte-h":!0}),w(de)!=="svelte-pvv6jb"&&(de.innerHTML=fl),gt=c(e),U(me.$$.fragment,e),Qt=c(e),Ue=T(e,"P",{"data-svelte-h":!0}),w(Ue)!=="svelte-bfmb2z"&&(Ue.textContent=Tl),Zt=c(e),U(ye.$$.fragment,e),Gt=c(e),U(Je.$$.fragment,e),Ct=c(e),be=T(e,"P",{"data-svelte-h":!0}),w(be)!=="svelte-197key7"&&(be.innerHTML=jl),It=c(e),U(fe.$$.fragment,e),kt=c(e),Te=T(e,"P",{"data-svelte-h":!0}),w(Te)!=="svelte-iwzotw"&&(Te.textContent=hl),$t=c(e),U(je.$$.fragment,e),Rt=c(e),he=T(e,"P",{"data-svelte-h":!0}),w(he)!=="svelte-1goq14a"&&(he.innerHTML=wl),_t=c(e),x.l(e),Ee=c(e),we=T(e,"P",{"data-svelte-h":!0}),w(we)!=="svelte-z1os81"&&(we.innerHTML=gl),Nt=c(e),U(ge.$$.fragment,e),xt=c(e),Qe=T(e,"P",{"data-svelte-h":!0}),w(Qe)!=="svelte-1ksn3i2"&&(Qe.textContent=Ql),Bt=c(e),z.l(e),Ae=c(e),Ze=T(e,"P",{"data-svelte-h":!0}),w(Ze)!=="svelte-ozuz78"&&(Ze.textContent=Zl),zt=c(e),U(Ge.$$.fragment,e),vt=c(e),Ce=T(e,"P",{"data-svelte-h":!0}),w(Ce)!=="svelte-kgage5"&&(Ce.innerHTML=Gl),Xt=c(e),Ie=T(e,"P",{"data-svelte-h":!0}),w(Ie)!=="svelte-n2rsrh"&&(Ie.innerHTML=Cl),Vt=c(e),U(ke.$$.fragment,e),Wt=c(e),$e=T(e,"P",{"data-svelte-h":!0}),w($e)!=="svelte-ply641"&&($e.innerHTML=Il),Yt=c(e),X.l(e),He=c(e),Re=T(e,"P",{"data-svelte-h":!0}),w(Re)!=="svelte-19g227w"&&(Re.textContent=kl),St=c(e),U(_e.$$.fragment,e),Ft=c(e),Ne=T(e,"P",{"data-svelte-h":!0}),w(Ne)!=="svelte-jyw2su"&&(Ne.innerHTML=$l),Et=c(e),U(xe.$$.fragment,e),At=c(e),Be=T(e,"P",{"data-svelte-h":!0}),w(Be)!=="svelte-15mcgtc"&&(Be.textContent=Rl),Ht=c(e),U(ze.$$.fragment,e),qt=c(e),U(ve.$$.fragment,e),Dt=c(e),Xe=T(e,"P",{"data-svelte-h":!0}),w(Xe)!=="svelte-19uhxe9"&&(Xe.textContent=_l),Lt=c(e),U(Y.$$.fragment,e),Pt=c(e),U(Ve.$$.fragment,e),Kt=c(e),qe=T(e,"P",{}),El(qe).forEach(l),this.h()},h(){De(a,"name","hf:doc:metadata"),De(a,"content",Us),De(V,"class","flex justify-center"),De(_,"border","1"),De(_,"class","dataframe"),il(_,"table-layout","fixed"),il(_,"word-wrap","break-word"),il(_,"width","100%")},m(e,t){es(document.head,a),s(e,o,t),s(e,n,t),s(e,h,t),y(u,e,t),s(e,d,t),y(G,e,t),s(e,b,t),We[Q].m(e,t),s(e,R,t),s(e,k,t),s(e,$,t),y(i,e,t),s(e,g,t),y(S,e,t),s(e,Pe,t),s(e,F,t),s(e,Ke,t),s(e,E,t),s(e,Oe,t),s(e,V,t),s(e,et,t),y(A,e,t),s(e,tt,t),s(e,H,t),s(e,lt,t),y(q,e,t),s(e,st,t),y(D,e,t),s(e,at,t),s(e,L,t),s(e,nt,t),y(P,e,t),s(e,it,t),y(K,e,t),s(e,rt,t),s(e,O,t),s(e,ct,t),y(ee,e,t),s(e,ut,t),y(te,e,t),s(e,Mt,t),s(e,le,t),s(e,pt,t),y(se,e,t),s(e,ot,t),s(e,ae,t),s(e,dt,t),y(ne,e,t),s(e,mt,t),y(ie,e,t),s(e,Ut,t),s(e,re,t),s(e,yt,t),y(ce,e,t),s(e,Jt,t),s(e,_,t),s(e,bt,t),s(e,ue,t),s(e,ft,t),y(Me,e,t),s(e,Tt,t),y(pe,e,t),s(e,jt,t),s(e,oe,t),s(e,ht,t),y(W,e,t),s(e,wt,t),s(e,de,t),s(e,gt,t),y(me,e,t),s(e,Qt,t),s(e,Ue,t),s(e,Zt,t),y(ye,e,t),s(e,Gt,t),y(Je,e,t),s(e,Ct,t),s(e,be,t),s(e,It,t),y(fe,e,t),s(e,kt,t),s(e,Te,t),s(e,$t,t),y(je,e,t),s(e,Rt,t),s(e,he,t),s(e,_t,t),Ye[N].m(e,t),s(e,Ee,t),s(e,we,t),s(e,Nt,t),y(ge,e,t),s(e,xt,t),s(e,Qe,t),s(e,Bt,t),Se[B].m(e,t),s(e,Ae,t),s(e,Ze,t),s(e,zt,t),y(Ge,e,t),s(e,vt,t),s(e,Ce,t),s(e,Xt,t),s(e,Ie,t),s(e,Vt,t),y(ke,e,t),s(e,Wt,t),s(e,$e,t),s(e,Yt,t),Fe[v].m(e,t),s(e,He,t),s(e,Re,t),s(e,St,t),y(_e,e,t),s(e,Ft,t),s(e,Ne,t),s(e,Et,t),y(xe,e,t),s(e,At,t),s(e,Be,t),s(e,Ht,t),y(ze,e,t),s(e,qt,t),y(ve,e,t),s(e,Dt,t),s(e,Xe,t),s(e,Lt,t),y(Y,e,t),s(e,Pt,t),y(Ve,e,t),s(e,Kt,t),s(e,qe,t),Ot=!0},p(e,[t]){const Yl={};t&1&&(Yl.fw=e[0]),u.$set(Yl);let ll=Q;Q=xl(e),Q!==ll&&(tl(),M(We[ll],1,1,()=>{We[ll]=null}),el(),Z=We[Q],Z||(Z=We[Q]=Nl[Q](e),Z.c()),p(Z,1),Z.m(R.parentNode,R));const Sl={};t&2&&(Sl.$$scope={dirty:t,ctx:e}),W.$set(Sl);let sl=N;N=zl(e),N!==sl&&(tl(),M(Ye[sl],1,1,()=>{Ye[sl]=null}),el(),x=Ye[N],x||(x=Ye[N]=Bl[N](e),x.c()),p(x,1),x.m(Ee.parentNode,Ee));let al=B;B=Xl(e),B!==al&&(tl(),M(Se[al],1,1,()=>{Se[al]=null}),el(),z=Se[B],z||(z=Se[B]=vl[B](e),z.c()),p(z,1),z.m(Ae.parentNode,Ae));let nl=v;v=Wl(e),v!==nl&&(tl(),M(Fe[nl],1,1,()=>{Fe[nl]=null}),el(),X=Fe[v],X||(X=Fe[v]=Vl[v](e),X.c()),p(X,1),X.m(He.parentNode,He));const Fl={};t&2&&(Fl.$$scope={dirty:t,ctx:e}),Y.$set(Fl)},i(e){Ot||(p(u.$$.fragment,e),p(G.$$.fragment,e),p(Z),p(i.$$.fragment,e),p(S.$$.fragment,e),p(A.$$.fragment,e),p(q.$$.fragment,e),p(D.$$.fragment,e),p(P.$$.fragment,e),p(K.$$.fragment,e),p(ee.$$.fragment,e),p(te.$$.fragment,e),p(se.$$.fragment,e),p(ne.$$.fragment,e),p(ie.$$.fragment,e),p(ce.$$.fragment,e),p(Me.$$.fragment,e),p(pe.$$.fragment,e),p(W.$$.fragment,e),p(me.$$.fragment,e),p(ye.$$.fragment,e),p(Je.$$.fragment,e),p(fe.$$.fragment,e),p(je.$$.fragment,e),p(x),p(ge.$$.fragment,e),p(z),p(Ge.$$.fragment,e),p(ke.$$.fragment,e),p(X),p(_e.$$.fragment,e),p(xe.$$.fragment,e),p(ze.$$.fragment,e),p(ve.$$.fragment,e),p(Y.$$.fragment,e),p(Ve.$$.fragment,e),Ot=!0)},o(e){M(u.$$.fragment,e),M(G.$$.fragment,e),M(Z),M(i.$$.fragment,e),M(S.$$.fragment,e),M(A.$$.fragment,e),M(q.$$.fragment,e),M(D.$$.fragment,e),M(P.$$.fragment,e),M(K.$$.fragment,e),M(ee.$$.fragment,e),M(te.$$.fragment,e),M(se.$$.fragment,e),M(ne.$$.fragment,e),M(ie.$$.fragment,e),M(ce.$$.fragment,e),M(Me.$$.fragment,e),M(pe.$$.fragment,e),M(W.$$.fragment,e),M(me.$$.fragment,e),M(ye.$$.fragment,e),M(Je.$$.fragment,e),M(fe.$$.fragment,e),M(je.$$.fragment,e),M(x),M(ge.$$.fragment,e),M(z),M(Ge.$$.fragment,e),M(ke.$$.fragment,e),M(X),M(_e.$$.fragment,e),M(xe.$$.fragment,e),M(ze.$$.fragment,e),M(ve.$$.fragment,e),M(Y.$$.fragment,e),M(Ve.$$.fragment,e),Ot=!1},d(e){e&&(l(o),l(n),l(h),l(d),l(b),l(R),l(k),l($),l(g),l(Pe),l(F),l(Ke),l(E),l(Oe),l(V),l(et),l(tt),l(H),l(lt),l(st),l(at),l(L),l(nt),l(it),l(rt),l(O),l(ct),l(ut),l(Mt),l(le),l(pt),l(ot),l(ae),l(dt),l(mt),l(Ut),l(re),l(yt),l(Jt),l(_),l(bt),l(ue),l(ft),l(Tt),l(jt),l(oe),l(ht),l(wt),l(de),l(gt),l(Qt),l(Ue),l(Zt),l(Gt),l(Ct),l(be),l(It),l(kt),l(Te),l($t),l(Rt),l(he),l(_t),l(Ee),l(we),l(Nt),l(xt),l(Qe),l(Bt),l(Ae),l(Ze),l(zt),l(vt),l(Ce),l(Xt),l(Ie),l(Vt),l(Wt),l($e),l(Yt),l(He),l(Re),l(St),l(Ft),l(Ne),l(Et),l(At),l(Be),l(Ht),l(qt),l(Dt),l(Xe),l(Lt),l(Pt),l(Kt),l(qe)),l(a),J(u,e),J(G,e),We[Q].d(e),J(i,e),J(S,e),J(A,e),J(q,e),J(D,e),J(P,e),J(K,e),J(ee,e),J(te,e),J(se,e),J(ne,e),J(ie,e),J(ce,e),J(Me,e),J(pe,e),J(W,e),J(me,e),J(ye,e),J(Je,e),J(fe,e),J(je,e),Ye[N].d(e),J(ge,e),Se[B].d(e),J(Ge,e),J(ke,e),Fe[v].d(e),J(_e,e),J(xe,e),J(ze,e),J(ve,e),J(Y,e),J(Ve,e)}}}const Us='{"title":"Căutare semantică cu FAISS","local":"semantic-search-with-faiss","sections":[{"title":"Utilizarea embeddings pentru căutare semantică","local":"using-embeddings-for-semantic-search","sections":[],"depth":2},{"title":"Încărcarea și pregătirea datasetului","local":"loading-and-preparing-the-dataset","sections":[],"depth":2},{"title":"Crearea embeddings-urilor de text","local":"creating-text-embeddings","sections":[],"depth":2},{"title":"Utilizarea FAISS pentru căutare de similaritate eficientă","local":"using-faiss-for-efficient-similarity-search","sections":[],"depth":2}],"depth":1}';function ys(C,a,o){let n="pt";return Ll(()=>{const h=new URLSearchParams(window.location.search);o(0,n=h.get("fw")||"pt")}),[n]}class Qs extends Pl{constructor(a){super(),Kl(this,a,ys,ms,Dl,{})}}export{Qs as component};
