import{s as wM,f as TM,n as cM,o as uM}from"../chunks/scheduler.65852ee5.js";import{S as rM,i as dM,g as n,s as M,r as J,A as IM,h as y,f as e,c as a,j as jM,u as i,x as U,k as os,y as mM,a as t,v as o,d as p,t as T,w as j}from"../chunks/index.aa74147d.js";import{D as hM,C as w}from"../chunks/DocNotebookDropdown.479f4286.js";import{H as c,E as CM}from"../chunks/EditOnGithub.4eda6a96.js";function bM(it){let u,js,ps,ws,d,cs,I,us,m,ot='<em>Authored by: <a href="https://huggingface.co/m-ric" rel="nofollow">Aymeric Roucher</a></em>',rs,h,pt="This notebook demonstrates how you can evaluate your RAG (Retrieval Augmented Generation), by building a synthetic evaluation dataset and using LLM-as-a-judge to compute the accuracy of your system.",ds,C,Tt='For an introduction to RAG, you can check <a href="rag_zephyr_langchain">this other cookbook</a>!',Is,b,jt="RAG systems are complex: here a RAG diagram, where we noted in blue all possibilities for system enhancement:",ms,Z,wt,hs,B,ct=`Implementing any of these improvements can bring a huge performance boost; but changing anything is useless if you cannot monitor the impact of your changes on the system’s performance!
So let’s see how to evaluate our RAG system.`,Cs,G,bs,A,ut="Since there are so many moving parts to tune with a big impact on performance, benchmarking the RAG system is crucial.",Zs,g,rt="For our evaluation pipeline, we will need:",Bs,f,dt="<li>An evaluation dataset with question - answer couples (QA couples)</li> <li>An evaluator to compute the accuracy of our system on the above evaluation dataset.</li>",Gs,W,It="➡️ It turns out, we can use LLMs to help us all along the way!",As,V,mt='<li>The evaluation dataset will be synthetically generated by an LLM 🤖, and questions will be filtered out by other LLMs 🤖</li> <li>An <a href="https://huggingface.co/papers/2306.05685" rel="nofollow">LLM-as-a-judge</a> agent 🤖 will then perform the evaluation on this synthetic dataset.</li>',gs,Q,ht="<strong>Let’s dig into it and start building our evaluation pipeline!</strong> First, we install the required model dependancies.",fs,N,Ws,R,Vs,v,Qs,X,Ns,k,Rs,E,vs,S,Xs,Y,Ct="We first build a synthetic dataset of questions and associated contexts. The method is to get elements from our knowledge base, and ask an LLM to generate questions based on these documents.",ks,q,bt="Then we setup other LLM agents to act as quality filters for the generated QA couples: each of them will act as the filter for a specific flaw.",Es,F,Ss,_,Ys,x,qs,z,Zt='We use <a href="https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1" rel="nofollow">Mixtral</a> for QA couple generation because it it has excellent performance in leaderboards such as <a href="https://huggingface.co/spaces/lmsys/chatbot-arena-leaderboard" rel="nofollow">Chatbot Arena</a>.',Fs,H,_s,$,xs,L,Bt=`Now let’s generate our QA couples.
For this example, we generate only 10 QA couples and will load the rest from the Hub.`,zs,D,Gt="But for your specific knowledge base, given that you want to get at least ~100 test samples, and accounting for the fact that we will filter out around half of these with our critique agents later on, you should generate much more, in the >200 samples.",Hs,K,$s,P,Ls,O,Ds,ll,At="The questions generated by the previous agent can have many flaws: we should do a quality check before validating these questions.",Ks,sl,gt='We thus build critique agents that will rate each question on several criteria, given in <a href="https://huggingface.co/papers/2312.10003" rel="nofollow">this paper</a>:',Ps,el,ft="<li><strong>Groundedness:</strong> can the question be answered from the given context?</li> <li><strong>Relevance:</strong> is the question relevant to users? For instance, <code>&quot;What is the date when transformers 4.29.1 was released?&quot;</code> is not relevant for ML practicioners.</li>",Os,tl,Wt=`One last failure case we’ve noticed is when a function is tailored for the particular setting where the question was generated, but undecipherable by itself, like <code>&quot;What is the name of the function used in this guide?&quot;</code>.
We also build a critique agent for this criteria:`,le,Ml,Vt="<li><strong>Stand-alone</strong>: is the question understandable free of any context, for someone with domain knowledge/Internet access? The opposite of this would be <code>What is the function used in this article?</code> for a question generated from a specific blog article.</li>",se,al,Qt="We systematically score functions with all these agents, and whenever the score is too low for any one of the agents, we eliminate the question from our eval dataset.",ee,nl,Nt="💡 <strong><em>When asking the agents to output a score, we first ask them to produce its rationale. This will help us verify scores, but most importantly, asking it to first output rationale gives the model more tokens to think and elaborate an answer before summarizing it into a single score token.</em></strong>",te,yl,Rt="We now build and run these critique agents.",Me,Ul,ae,Jl,ne,il,vt="Now let us filter out bad questions based on our critique agent scores:",ye,ol,Ue,pl,Xt=`Evaluation dataset before filtering:
`,Je,Tl,kt="Now our synthetic evaluation dataset is complete! We can evaluate different RAG systems on this evaluation dataset.",ie,jl,Et="We have generated only a few QA couples here to reduce time and cost. But let’s kick start the next part by loading a pre-generated dataset:",oe,wl,pe,cl,Te,ul,je,rl,St="<li>In this part, <strong>we split the documents from our knowledge base into smaller chunks</strong>: these will be the snippets that are picked by the Retriever, to then be ingested by the Reader LLM as supporting elements for its answer.</li> <li>The goal is to build semantically relevant snippets: not too small to be sufficient for supporting an answer, and not too large too avoid diluting individual ideas.</li>",we,dl,Yt="Many options exist for text splitting:",ce,Il,qt="<li>split every <code>n</code> words / characters, but this has the risk of cutting in half paragraphs or even sentences</li> <li>split after <code>n</code> words / character, but only on sentence boundaries</li> <li><strong>recursive split</strong> tries to preserve even more of the document structure, by processing it tree-like way, splitting first on the largest units (chapters) then recursively splitting on smaller units (paragraphs, sentences).</li>",ue,ml,Ft='To learn more about chunking, I recommend you read <a href="https://github.com/FullStackRetrieval-com/RetrievalTutorials/blob/main/tutorials/LevelsOfTextSplitting/5_Levels_Of_Text_Splitting.ipynb" rel="nofollow">this great notebook</a> by Greg Kamradt.',re,hl,_t='<a href="https://huggingface.co/spaces/m-ric/chunk_visualizer" rel="nofollow">This space</a> lets you visualize how different splitting options affect the chunks you get.',de,Cl,xt="<p>In the following, we use Langchain’s <code>RecursiveCharacterTextSplitter</code>.</p>",Ie,bl,zt="💡 <em>To measure chunk length in our Text Splitter, our length function will not be the count of characters, but the count of tokens in the tokenized text: indeed, for subsequent embedder that processes token, measuring length in tokens is more relevant and empirically performs better.</em>",me,Zl,he,Bl,Ce,Gl,be,Al,Ht="The <strong>retriever acts like an internal search engine</strong>: given the user query, it returns the most relevant documents from your knowledge base.",Ze,gl,$t='<p>For the knowledge base, we use Langchain vector databases since <strong>it offers a convenient <a href="https://github.com/facebookresearch/faiss" rel="nofollow">FAISS</a> index and allows us to keep document metadata throughout the processing</strong>.</p>',Be,fl,Lt="🛠️ <strong>Options included:</strong>",Ge,Wl,Dt='<li>Tune the chunking method:<ul><li>Size of the chunks</li> <li>Method: split on different separators, use <a href="https://python.langchain.com/docs/modules/data_connection/document_transformers/semantic-chunker" rel="nofollow">semantic chunking</a>…</li></ul></li> <li>Change the embedding model</li>',Ae,Vl,ge,Ql,fe,Nl,Kt="In this part, the <strong>LLM Reader reads the retrieved documents to formulate its answer.</strong>",We,Rl,Pt="🛠️ Here we tried the following options to improve results:",Ve,vl,Ot="<li>Switch reranking on/off</li> <li>Change the reader model</li>",Qe,Xl,Ne,kl,Re,El,ve,Sl,Xe,Yl,lM="The RAG system and the evaluation datasets are now ready. The last step is to judge the RAG system’s output on this evlauation dataset.",ke,ql,sM="To this end, <strong>we setup a judge agent</strong>. ⚖️🤖",Ee,Fl,eM='Out of <a href="https://docs.ragas.io/en/latest/concepts/metrics/index.html" rel="nofollow">the different RAG evaluation metrics</a>, we choose to focus only on faithfulness since it the best end-to-end metric of our system’s performance.',Se,_l,tM='<p>We use GPT4 as a judge for its empirically good performance, but you could try with other models such as <a href="https://huggingface.co/kaist-ai/prometheus-13b-v1.0" rel="nofollow">kaist-ai/prometheus-13b-v1.0</a> or <a href="https://huggingface.co/BAAI/JudgeLM-33B-v1.0" rel="nofollow">BAAI/JudgeLM-33B-v1.0</a>.</p>',Ye,xl,MM='💡 <em>In the evaluation prompt, we give a detailed description each metric on the scale 1-5, as is done in <a href="https://huggingface.co/kaist-ai/prometheus-13b-v1.0" rel="nofollow">Prometheus’s prompt template</a>: this helps the model ground its metric precisely. If instead you give the judge LLM a vague scale to work with, the outputs will not be consistent enough between different examples.</em>',qe,zl,aM="💡 <em>Again, prompting the LLM to output rationale before giving its final score gives it more tokens to help it formalize and elaborate a judgement.</em>",Fe,Hl,_e,$l,xe,Ll,ze,Dl,nM="🚀 Let’s run the tests and evaluate answers!👇",He,Kl,$e,Pl,Le,Ol,De,ls,Ke,ss,Pe,es,Oe,ts,yM=`Let us load the results that I obtained by tweaking the different options available in this notebook.
For more detail on why these options could work on not, see the notebook on <a href="advanced_rag">advanced_RAG</a>.`,lt,Ms,UM="As you can see in the graph below, some tweaks do not bring any improvement, some give huge performance boosts.",st,as,JM="➡️ <strong><em>There is no single good recipe: you should try several different directions when tuning your RAG systems.</em></strong>",et,ns,tt,ys,Mt,r,iM,at,Us,oM="As you can see, these had varying impact on performance. In particular, tuning the chunk size is both easy and very impactful.",nt,Js,pM="But this is our case: your results could be very different: now that you have a robust evaluation pipeline, you can set on to explore other options! 🗺️",yt,is,Ut,Ts,Jt;return d=new hM({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/cookbook/blob/main/notebooks/en/rag_evaluation.ipynb"}]}}),I=new c({props:{title:"RAG Evaluation",local:"rag-evaluation",headingTag:"h1"}}),G=new c({props:{title:"Evaluating RAG performance",local:"evaluating-rag-performance",headingTag:"h3"}}),N=new w({props:{code:"IXBpcCUyMGluc3RhbGwlMjAtcSUyMHRvcmNoJTIwdHJhbnNmb3JtZXJzJTIwdHJhbnNmb3JtZXJzJTIwbGFuZ2NoYWluJTIwc2VudGVuY2UtdHJhbnNmb3JtZXJzJTIwdHFkbSUyMG9wZW5weXhsJTIwb3BlbmFpJTIwcGFuZGFzJTIwZGF0YXNldHMlMjBsYW5nY2hhaW4tY29tbXVuaXR5JTIwcmFnYXRvdWlsbGU=",highlighted:"!pip install -q torch transformers transformers langchain sentence-transformers tqdm openpyxl openai pandas datasets langchain-community ragatouille",wrap:!1}}),R=new w({props:{code:"JTI1cmVsb2FkX2V4dCUyMGF1dG9yZWxvYWQlMEElMjVhdXRvcmVsb2FkJTIwMg==",highlighted:`%reload_ext autoreload
%autoreload <span class="hljs-number">2</span>`,wrap:!1}}),v=new w({props:{code:"ZnJvbSUyMHRxZG0uYXV0byUyMGltcG9ydCUyMHRxZG0lMEFpbXBvcnQlMjBwYW5kYXMlMjBhcyUyMHBkJTBBZnJvbSUyMHR5cGluZyUyMGltcG9ydCUyME9wdGlvbmFsJTJDJTIwTGlzdCUyQyUyMFR1cGxlJTBBaW1wb3J0JTIwanNvbiUwQWltcG9ydCUyMGRhdGFzZXRzJTBBJTBBcGQuc2V0X29wdGlvbiglMjJkaXNwbGF5Lm1heF9jb2x3aWR0aCUyMiUyQyUyME5vbmUp",highlighted:`<span class="hljs-keyword">from</span> tqdm.auto <span class="hljs-keyword">import</span> tqdm
<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
<span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> <span class="hljs-type">Optional</span>, <span class="hljs-type">List</span>, <span class="hljs-type">Tuple</span>
<span class="hljs-keyword">import</span> json
<span class="hljs-keyword">import</span> datasets

pd.set_option(<span class="hljs-string">&quot;display.max_colwidth&quot;</span>, <span class="hljs-literal">None</span>)`,wrap:!1}}),X=new w({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`,wrap:!1}}),k=new c({props:{title:"Load your knowledge base",local:"load-your-knowledge-base",headingTag:"h3"}}),E=new w({props:{code:"ZHMlMjAlM0QlMjBkYXRhc2V0cy5sb2FkX2RhdGFzZXQoJTIybS1yaWMlMkZodWdnaW5nZmFjZV9kb2MlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKQ==",highlighted:'ds = datasets.load_dataset(<span class="hljs-string">&quot;m-ric/huggingface_doc&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)',wrap:!1}}),S=new c({props:{title:"1. Build a synthetic dataset for evaluation",local:"1-build-a-synthetic-dataset-for-evaluation",headingTag:"h1"}}),F=new c({props:{title:"1.1. Prepare source documents",local:"11-prepare-source-documents",headingTag:"h3"}}),_=new w({props:{code:"ZnJvbSUyMGxhbmdjaGFpbi50ZXh0X3NwbGl0dGVyJTIwaW1wb3J0JTIwUmVjdXJzaXZlQ2hhcmFjdGVyVGV4dFNwbGl0dGVyJTBBZnJvbSUyMGxhbmdjaGFpbi5kb2NzdG9yZS5kb2N1bWVudCUyMGltcG9ydCUyMERvY3VtZW50JTIwYXMlMjBMYW5nY2hhaW5Eb2N1bWVudCUwQSUwQWxhbmdjaGFpbl9kb2NzJTIwJTNEJTIwJTVCTGFuZ2NoYWluRG9jdW1lbnQocGFnZV9jb250ZW50JTNEZG9jJTVCJTIydGV4dCUyMiU1RCUyQyUyMG1ldGFkYXRhJTNEJTdCJTIyc291cmNlJTIyJTNBJTIwZG9jJTVCJTIyc291cmNlJTIyJTVEJTdEKSUyMGZvciUyMGRvYyUyMGluJTIwdHFkbShkcyklNUQlMEElMEElMEF0ZXh0X3NwbGl0dGVyJTIwJTNEJTIwUmVjdXJzaXZlQ2hhcmFjdGVyVGV4dFNwbGl0dGVyKCUwQSUyMCUyMCUyMCUyMGNodW5rX3NpemUlM0QyMDAwJTJDJTBBJTIwJTIwJTIwJTIwY2h1bmtfb3ZlcmxhcCUzRDIwMCUyQyUwQSUyMCUyMCUyMCUyMGFkZF9zdGFydF9pbmRleCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBzZXBhcmF0b3JzJTNEJTVCJTIyJTVDbiU1Q24lMjIlMkMlMjAlMjIlNUNuJTIyJTJDJTIwJTIyLiUyMiUyQyUyMCUyMiUyMCUyMiUyQyUyMCUyMiUyMiU1RCUyQyUwQSklMEElMEFkb2NzX3Byb2Nlc3NlZCUyMCUzRCUyMCU1QiU1RCUwQWZvciUyMGRvYyUyMGluJTIwbGFuZ2NoYWluX2RvY3MlM0ElMEElMjAlMjAlMjAlMjBkb2NzX3Byb2Nlc3NlZCUyMCUyQiUzRCUyMHRleHRfc3BsaXR0ZXIuc3BsaXRfZG9jdW1lbnRzKCU1QmRvYyU1RCk=",highlighted:`<span class="hljs-keyword">from</span> langchain.text_splitter <span class="hljs-keyword">import</span> RecursiveCharacterTextSplitter
<span class="hljs-keyword">from</span> langchain.docstore.document <span class="hljs-keyword">import</span> Document <span class="hljs-keyword">as</span> LangchainDocument

langchain_docs = [LangchainDocument(page_content=doc[<span class="hljs-string">&quot;text&quot;</span>], metadata={<span class="hljs-string">&quot;source&quot;</span>: doc[<span class="hljs-string">&quot;source&quot;</span>]}) <span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> tqdm(ds)]


text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=<span class="hljs-number">2000</span>,
    chunk_overlap=<span class="hljs-number">200</span>,
    add_start_index=<span class="hljs-literal">True</span>,
    separators=[<span class="hljs-string">&quot;\\n\\n&quot;</span>, <span class="hljs-string">&quot;\\n&quot;</span>, <span class="hljs-string">&quot;.&quot;</span>, <span class="hljs-string">&quot; &quot;</span>, <span class="hljs-string">&quot;&quot;</span>],
)

docs_processed = []
<span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> langchain_docs:
    docs_processed += text_splitter.split_documents([doc])`,wrap:!1}}),x=new c({props:{title:"1.2. Setup agents for question generation",local:"12-setup-agents-for-question-generation",headingTag:"h3"}}),H=new w({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEluZmVyZW5jZUNsaWVudCUwQSUwQSUwQXJlcG9faWQlMjAlM0QlMjAlMjJtaXN0cmFsYWklMkZNaXh0cmFsLTh4N0ItSW5zdHJ1Y3QtdjAuMSUyMiUwQSUwQWxsbV9jbGllbnQlMjAlM0QlMjBJbmZlcmVuY2VDbGllbnQoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0RyZXBvX2lkJTJDJTBBJTIwJTIwJTIwJTIwdGltZW91dCUzRDEyMCUyQyUwQSklMEElMEElMEFkZWYlMjBjYWxsX2xsbShpbmZlcmVuY2VfY2xpZW50JTNBJTIwSW5mZXJlbmNlQ2xpZW50JTJDJTIwcHJvbXB0JTNBJTIwc3RyKSUzQSUwQSUyMCUyMCUyMCUyMHJlc3BvbnNlJTIwJTNEJTIwaW5mZXJlbmNlX2NsaWVudC5wb3N0KCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGpzb24lM0QlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJpbnB1dHMlMjIlM0ElMjBwcm9tcHQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJwYXJhbWV0ZXJzJTIyJTNBJTIwJTdCJTIybWF4X25ld190b2tlbnMlMjIlM0ElMjAxMDAwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydGFzayUyMiUzQSUyMCUyMnRleHQtZ2VuZXJhdGlvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjByZXR1cm4lMjBqc29uLmxvYWRzKHJlc3BvbnNlLmRlY29kZSgpKSU1QjAlNUQlNUIlMjJnZW5lcmF0ZWRfdGV4dCUyMiU1RCUwQSUwQSUwQWNhbGxfbGxtKGxsbV9jbGllbnQlMkMlMjAlMjJUaGlzJTIwaXMlMjBhJTIwdGVzdCUyMGNvbnRleHQlMjIp",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient


repo_id = <span class="hljs-string">&quot;mistralai/Mixtral-8x7B-Instruct-v0.1&quot;</span>

llm_client = InferenceClient(
    model=repo_id,
    timeout=<span class="hljs-number">120</span>,
)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">call_llm</span>(<span class="hljs-params">inference_client: InferenceClient, prompt: <span class="hljs-built_in">str</span></span>):
    response = inference_client.post(
        json={
            <span class="hljs-string">&quot;inputs&quot;</span>: prompt,
            <span class="hljs-string">&quot;parameters&quot;</span>: {<span class="hljs-string">&quot;max_new_tokens&quot;</span>: <span class="hljs-number">1000</span>},
            <span class="hljs-string">&quot;task&quot;</span>: <span class="hljs-string">&quot;text-generation&quot;</span>,
        },
    )
    <span class="hljs-keyword">return</span> json.loads(response.decode())[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;generated_text&quot;</span>]


call_llm(llm_client, <span class="hljs-string">&quot;This is a test context&quot;</span>)`,wrap:!1}}),$=new w({props:{code:"UUFfZ2VuZXJhdGlvbl9wcm9tcHQlMjAlM0QlMjAlMjIlMjIlMjIlMEFZb3VyJTIwdGFzayUyMGlzJTIwdG8lMjB3cml0ZSUyMGElMjBmYWN0b2lkJTIwcXVlc3Rpb24lMjBhbmQlMjBhbiUyMGFuc3dlciUyMGdpdmVuJTIwYSUyMGNvbnRleHQuJTBBWW91ciUyMGZhY3RvaWQlMjBxdWVzdGlvbiUyMHNob3VsZCUyMGJlJTIwYW5zd2VyYWJsZSUyMHdpdGglMjBhJTIwc3BlY2lmaWMlMkMlMjBjb25jaXNlJTIwcGllY2UlMjBvZiUyMGZhY3R1YWwlMjBpbmZvcm1hdGlvbiUyMGZyb20lMjB0aGUlMjBjb250ZXh0LiUwQVlvdXIlMjBmYWN0b2lkJTIwcXVlc3Rpb24lMjBzaG91bGQlMjBiZSUyMGZvcm11bGF0ZWQlMjBpbiUyMHRoZSUyMHNhbWUlMjBzdHlsZSUyMGFzJTIwcXVlc3Rpb25zJTIwdXNlcnMlMjBjb3VsZCUyMGFzayUyMGluJTIwYSUyMHNlYXJjaCUyMGVuZ2luZS4lMEFUaGlzJTIwbWVhbnMlMjB0aGF0JTIweW91ciUyMGZhY3RvaWQlMjBxdWVzdGlvbiUyME1VU1QlMjBOT1QlMjBtZW50aW9uJTIwc29tZXRoaW5nJTIwbGlrZSUyMCUyMmFjY29yZGluZyUyMHRvJTIwdGhlJTIwcGFzc2FnZSUyMiUyMG9yJTIwJTIyY29udGV4dCUyMi4lMEElMEFQcm92aWRlJTIweW91ciUyMGFuc3dlciUyMGFzJTIwZm9sbG93cyUzQSUwQSUwQU91dHB1dCUzQSUzQSUzQSUwQUZhY3RvaWQlMjBxdWVzdGlvbiUzQSUyMCh5b3VyJTIwZmFjdG9pZCUyMHF1ZXN0aW9uKSUwQUFuc3dlciUzQSUyMCh5b3VyJTIwYW5zd2VyJTIwdG8lMjB0aGUlMjBmYWN0b2lkJTIwcXVlc3Rpb24pJTBBJTBBTm93JTIwaGVyZSUyMGlzJTIwdGhlJTIwY29udGV4dC4lMEElMEFDb250ZXh0JTNBJTIwJTdCY29udGV4dCU3RCU1Q24lMEFPdXRwdXQlM0ElM0ElM0ElMjIlMjIlMjI=",highlighted:`QA_generation_prompt = <span class="hljs-string">&quot;&quot;&quot;
Your task is to write a factoid question and an answer given a context.
Your factoid question should be answerable with a specific, concise piece of factual information from the context.
Your factoid question should be formulated in the same style as questions users could ask in a search engine.
This means that your factoid question MUST NOT mention something like &quot;according to the passage&quot; or &quot;context&quot;.

Provide your answer as follows:

Output:::
Factoid question: (your factoid question)
Answer: (your answer to the factoid question)

Now here is the context.

Context: {context}\\n
Output:::&quot;&quot;&quot;</span>`,wrap:!1}}),K=new w({props:{code:"aW1wb3J0JTIwcmFuZG9tJTBBJTBBTl9HRU5FUkFUSU9OUyUyMCUzRCUyMDEwJTIwJTIwJTIzJTIwV2UlMjBpbnRlbnRpb25hbGx5JTIwZ2VuZXJhdGUlMjBvbmx5JTIwMTAlMjBRQSUyMGNvdXBsZXMlMjBoZXJlJTIwZm9yJTIwY29zdCUyMGFuZCUyMHRpbWUlMjBjb25zaWRlcmF0aW9ucyUwQSUwQXByaW50KGYlMjJHZW5lcmF0aW5nJTIwJTdCTl9HRU5FUkFUSU9OUyU3RCUyMFFBJTIwY291cGxlcy4uLiUyMiklMEElMEFvdXRwdXRzJTIwJTNEJTIwJTVCJTVEJTBBZm9yJTIwc2FtcGxlZF9jb250ZXh0JTIwaW4lMjB0cWRtKHJhbmRvbS5zYW1wbGUoZG9jc19wcm9jZXNzZWQlMkMlMjBOX0dFTkVSQVRJT05TKSklM0ElMEElMjAlMjAlMjAlMjAlMjMlMjBHZW5lcmF0ZSUyMFFBJTIwY291cGxlJTBBJTIwJTIwJTIwJTIwb3V0cHV0X1FBX2NvdXBsZSUyMCUzRCUyMGNhbGxfbGxtKGxsbV9jbGllbnQlMkMlMjBRQV9nZW5lcmF0aW9uX3Byb21wdC5mb3JtYXQoY29udGV4dCUzRHNhbXBsZWRfY29udGV4dC5wYWdlX2NvbnRlbnQpKSUwQSUyMCUyMCUyMCUyMHRyeSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHF1ZXN0aW9uJTIwJTNEJTIwb3V0cHV0X1FBX2NvdXBsZS5zcGxpdCglMjJGYWN0b2lkJTIwcXVlc3Rpb24lM0ElMjAlMjIpJTVCLTElNUQuc3BsaXQoJTIyQW5zd2VyJTNBJTIwJTIyKSU1QjAlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhbnN3ZXIlMjAlM0QlMjBvdXRwdXRfUUFfY291cGxlLnNwbGl0KCUyMkFuc3dlciUzQSUyMCUyMiklNUItMSU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFzc2VydCUyMGxlbihhbnN3ZXIpJTIwJTNDJTIwMzAwJTJDJTIwJTIyQW5zd2VyJTIwaXMlMjB0b28lMjBsb25nJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0cy5hcHBlbmQoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGV4dCUyMiUzQSUyMHNhbXBsZWRfY29udGV4dC5wYWdlX2NvbnRlbnQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJxdWVzdGlvbiUyMiUzQSUyMHF1ZXN0aW9uJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYW5zd2VyJTIyJTNBJTIwYW5zd2VyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyc291cmNlX2RvYyUyMiUzQSUyMHNhbXBsZWRfY29udGV4dC5tZXRhZGF0YSU1QiUyMnNvdXJjZSUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjBleGNlcHQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb250aW51ZQ==",highlighted:`<span class="hljs-keyword">import</span> random

N_GENERATIONS = <span class="hljs-number">10</span>  <span class="hljs-comment"># We intentionally generate only 10 QA couples here for cost and time considerations</span>

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Generating <span class="hljs-subst">{N_GENERATIONS}</span> QA couples...&quot;</span>)

outputs = []
<span class="hljs-keyword">for</span> sampled_context <span class="hljs-keyword">in</span> tqdm(random.sample(docs_processed, N_GENERATIONS)):
    <span class="hljs-comment"># Generate QA couple</span>
    output_QA_couple = call_llm(llm_client, QA_generation_prompt.<span class="hljs-built_in">format</span>(context=sampled_context.page_content))
    <span class="hljs-keyword">try</span>:
        question = output_QA_couple.split(<span class="hljs-string">&quot;Factoid question: &quot;</span>)[-<span class="hljs-number">1</span>].split(<span class="hljs-string">&quot;Answer: &quot;</span>)[<span class="hljs-number">0</span>]
        answer = output_QA_couple.split(<span class="hljs-string">&quot;Answer: &quot;</span>)[-<span class="hljs-number">1</span>]
        <span class="hljs-keyword">assert</span> <span class="hljs-built_in">len</span>(answer) &lt; <span class="hljs-number">300</span>, <span class="hljs-string">&quot;Answer is too long&quot;</span>
        outputs.append(
            {
                <span class="hljs-string">&quot;context&quot;</span>: sampled_context.page_content,
                <span class="hljs-string">&quot;question&quot;</span>: question,
                <span class="hljs-string">&quot;answer&quot;</span>: answer,
                <span class="hljs-string">&quot;source_doc&quot;</span>: sampled_context.metadata[<span class="hljs-string">&quot;source&quot;</span>],
            }
        )
    <span class="hljs-keyword">except</span>:
        <span class="hljs-keyword">continue</span>`,wrap:!1}}),P=new w({props:{code:"ZGlzcGxheShwZC5EYXRhRnJhbWUob3V0cHV0cykuaGVhZCgxKSk=",highlighted:'display(pd.DataFrame(outputs).head(<span class="hljs-number">1</span>))',wrap:!1}}),O=new c({props:{title:"1.3. Setup critique agents",local:"13-setup-critique-agents",headingTag:"h3"}}),Ul=new w({props:{code:" ",highlighted:`question_groundedness_critique_prompt = <span class="hljs-string">&quot;&quot;&quot;
You will be given a context and a question.
Your task is to provide a &#x27;total rating&#x27; scoring how well one can answer the given question unambiguously with the given context.
Give your answer on a scale of 1 to 5, where 1 means that the question is not answerable at all given the context, and 5 means that the question is clearly and unambiguously answerable with the context.

Provide your answer as follows:

Answer:::
Evaluation: (your rationale for the rating, as a text)
Total rating: (your rating, as a number between 1 and 5)

You MUST provide values for &#x27;Evaluation:&#x27; and &#x27;Total rating:&#x27; in your answer.

Now here are the question and context.

Question: {question}\\n
Context: {context}\\n
Answer::: &quot;&quot;&quot;</span>

question_relevance_critique_prompt = <span class="hljs-string">&quot;&quot;&quot;
You will be given a question.
Your task is to provide a &#x27;total rating&#x27; representing how useful this question can be to machine learning developers building NLP applications with the Hugging Face ecosystem.
Give your answer on a scale of 1 to 5, where 1 means that the question is not useful at all, and 5 means that the question is extremely useful.

Provide your answer as follows:

Answer:::
Evaluation: (your rationale for the rating, as a text)
Total rating: (your rating, as a number between 1 and 5)

You MUST provide values for &#x27;Evaluation:&#x27; and &#x27;Total rating:&#x27; in your answer.

Now here is the question.

Question: {question}\\n
Answer::: &quot;&quot;&quot;</span>

question_standalone_critique_prompt = <span class="hljs-string">&quot;&quot;&quot;
You will be given a question.
Your task is to provide a &#x27;total rating&#x27; representing how context-independant this question is.
Give your answer on a scale of 1 to 5, where 1 means that the question depends on additional information to be understood, and 5 means that the question makes sense by itself.
For instance, if the question refers to a particular setting, like &#x27;in the context&#x27; or &#x27;in the document&#x27;, the rating must be 1.
The questions can contain obscure technical nouns or acronyms like Gradio, Hub, Hugging Face or Space and still be a 5: it must simply be clear to an operator with access to documentation what the question is about.

For instance, &quot;What is the name of the checkpoint from which the ViT model is imported?&quot; should receive a 1, since there is an implicit mention of a context, thus the question is not independant from the context.

Provide your answer as follows:

Answer:::
Evaluation: (your rationale for the rating, as a text)
Total rating: (your rating, as a number between 1 and 5)

You MUST provide values for &#x27;Evaluation:&#x27; and &#x27;Total rating:&#x27; in your answer.

Now here is the question.

Question: {question}\\n
Answer::: &quot;&quot;&quot;</span>`,wrap:!1}}),Jl=new w({props:{code:"cHJpbnQoJTIyR2VuZXJhdGluZyUyMGNyaXRpcXVlJTIwZm9yJTIwZWFjaCUyMFFBJTIwY291cGxlLi4uJTIyKSUwQWZvciUyMG91dHB1dCUyMGluJTIwdHFkbShvdXRwdXRzKSUzQSUwQSUyMCUyMCUyMCUyMGV2YWx1YXRpb25zJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZ3JvdW5kZWRuZXNzJTIyJTNBJTIwY2FsbF9sbG0oJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGxtX2NsaWVudCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHF1ZXN0aW9uX2dyb3VuZGVkbmVzc19jcml0aXF1ZV9wcm9tcHQuZm9ybWF0KGNvbnRleHQlM0RvdXRwdXQlNUIlMjJjb250ZXh0JTIyJTVEJTJDJTIwcXVlc3Rpb24lM0RvdXRwdXQlNUIlMjJxdWVzdGlvbiUyMiU1RCklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjApJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycmVsZXZhbmNlJTIyJTNBJTIwY2FsbF9sbG0oJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGxtX2NsaWVudCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHF1ZXN0aW9uX3JlbGV2YW5jZV9jcml0aXF1ZV9wcm9tcHQuZm9ybWF0KHF1ZXN0aW9uJTNEb3V0cHV0JTVCJTIycXVlc3Rpb24lMjIlNUQpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YW5kYWxvbmUlMjIlM0ElMjBjYWxsX2xsbSglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsbG1fY2xpZW50JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcXVlc3Rpb25fc3RhbmRhbG9uZV9jcml0aXF1ZV9wcm9tcHQuZm9ybWF0KHF1ZXN0aW9uJTNEb3V0cHV0JTVCJTIycXVlc3Rpb24lMjIlNUQpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUyQyUwQSUyMCUyMCUyMCUyMCU3RCUwQSUyMCUyMCUyMCUyMHRyeSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZvciUyMGNyaXRlcmlvbiUyQyUyMGV2YWx1YXRpb24lMjBpbiUyMGV2YWx1YXRpb25zLml0ZW1zKCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzY29yZSUyQyUyMGV2YWwlMjAlM0QlMjAoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW50KGV2YWx1YXRpb24uc3BsaXQoJTIyVG90YWwlMjByYXRpbmclM0ElMjAlMjIpJTVCLTElNUQuc3RyaXAoKSklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBldmFsdWF0aW9uLnNwbGl0KCUyMlRvdGFsJTIwcmF0aW5nJTNBJTIwJTIyKSU1Qi0yJTVELnNwbGl0KCUyMkV2YWx1YXRpb24lM0ElMjAlMjIpJTVCMSU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXQudXBkYXRlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGYlMjIlN0Jjcml0ZXJpb24lN0Rfc2NvcmUlMjIlM0ElMjBzY29yZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGYlMjIlN0Jjcml0ZXJpb24lN0RfZXZhbCUyMiUzQSUyMGV2YWwlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwZXhjZXB0JTIwRXhjZXB0aW9uJTIwYXMlMjBlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY29udGludWU=",highlighted:`<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Generating critique for each QA couple...&quot;</span>)
<span class="hljs-keyword">for</span> output <span class="hljs-keyword">in</span> tqdm(outputs):
    evaluations = {
        <span class="hljs-string">&quot;groundedness&quot;</span>: call_llm(
            llm_client,
            question_groundedness_critique_prompt.<span class="hljs-built_in">format</span>(context=output[<span class="hljs-string">&quot;context&quot;</span>], question=output[<span class="hljs-string">&quot;question&quot;</span>]),
        ),
        <span class="hljs-string">&quot;relevance&quot;</span>: call_llm(
            llm_client,
            question_relevance_critique_prompt.<span class="hljs-built_in">format</span>(question=output[<span class="hljs-string">&quot;question&quot;</span>]),
        ),
        <span class="hljs-string">&quot;standalone&quot;</span>: call_llm(
            llm_client,
            question_standalone_critique_prompt.<span class="hljs-built_in">format</span>(question=output[<span class="hljs-string">&quot;question&quot;</span>]),
        ),
    }
    <span class="hljs-keyword">try</span>:
        <span class="hljs-keyword">for</span> criterion, evaluation <span class="hljs-keyword">in</span> evaluations.items():
            score, <span class="hljs-built_in">eval</span> = (
                <span class="hljs-built_in">int</span>(evaluation.split(<span class="hljs-string">&quot;Total rating: &quot;</span>)[-<span class="hljs-number">1</span>].strip()),
                evaluation.split(<span class="hljs-string">&quot;Total rating: &quot;</span>)[-<span class="hljs-number">2</span>].split(<span class="hljs-string">&quot;Evaluation: &quot;</span>)[<span class="hljs-number">1</span>],
            )
            output.update(
                {
                    <span class="hljs-string">f&quot;<span class="hljs-subst">{criterion}</span>_score&quot;</span>: score,
                    <span class="hljs-string">f&quot;<span class="hljs-subst">{criterion}</span>_eval&quot;</span>: <span class="hljs-built_in">eval</span>,
                }
            )
    <span class="hljs-keyword">except</span> Exception <span class="hljs-keyword">as</span> e:
        <span class="hljs-keyword">continue</span>`,wrap:!1}}),ol=new w({props:{code:"aW1wb3J0JTIwcGFuZGFzJTIwYXMlMjBwZCUwQSUwQXBkLnNldF9vcHRpb24oJTIyZGlzcGxheS5tYXhfY29sd2lkdGglMjIlMkMlMjBOb25lKSUwQSUwQWdlbmVyYXRlZF9xdWVzdGlvbnMlMjAlM0QlMjBwZC5EYXRhRnJhbWUuZnJvbV9kaWN0KG91dHB1dHMpJTBBJTBBcHJpbnQoJTIyRXZhbHVhdGlvbiUyMGRhdGFzZXQlMjBiZWZvcmUlMjBmaWx0ZXJpbmclM0ElMjIpJTBBZGlzcGxheSglMEElMjAlMjAlMjAlMjBnZW5lcmF0ZWRfcXVlc3Rpb25zJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycXVlc3Rpb24lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJhbnN3ZXIlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJncm91bmRlZG5lc3Nfc2NvcmUlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyZWxldmFuY2Vfc2NvcmUlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFuZGFsb25lX3Njb3JlJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVEJTBBJTIwJTIwJTIwJTIwJTVEJTBBKSUwQWdlbmVyYXRlZF9xdWVzdGlvbnMlMjAlM0QlMjBnZW5lcmF0ZWRfcXVlc3Rpb25zLmxvYyU1QiUwQSUyMCUyMCUyMCUyMChnZW5lcmF0ZWRfcXVlc3Rpb25zJTVCJTIyZ3JvdW5kZWRuZXNzX3Njb3JlJTIyJTVEJTIwJTNFJTNEJTIwNCklMEElMjAlMjAlMjAlMjAlMjYlMjAoZ2VuZXJhdGVkX3F1ZXN0aW9ucyU1QiUyMnJlbGV2YW5jZV9zY29yZSUyMiU1RCUyMCUzRSUzRCUyMDQpJTBBJTIwJTIwJTIwJTIwJTI2JTIwKGdlbmVyYXRlZF9xdWVzdGlvbnMlNUIlMjJzdGFuZGFsb25lX3Njb3JlJTIyJTVEJTIwJTNFJTNEJTIwNCklMEElNUQlMEFwcmludCglMjIlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlMjIpJTBBcHJpbnQoJTIyRmluYWwlMjBldmFsdWF0aW9uJTIwZGF0YXNldCUzQSUyMiklMEFkaXNwbGF5KCUwQSUyMCUyMCUyMCUyMGdlbmVyYXRlZF9xdWVzdGlvbnMlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJxdWVzdGlvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmFuc3dlciUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmdyb3VuZGVkbmVzc19zY29yZSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJlbGV2YW5jZV9zY29yZSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YW5kYWxvbmVfc2NvcmUlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUQlMEElMjAlMjAlMjAlMjAlNUQlMEEpJTBBJTBBZXZhbF9kYXRhc2V0JTIwJTNEJTIwZGF0YXNldHMuRGF0YXNldC5mcm9tX3BhbmRhcyhnZW5lcmF0ZWRfcXVlc3Rpb25zJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiUyQyUyMHByZXNlcnZlX2luZGV4JTNERmFsc2Up",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd

<span class="hljs-meta">&gt;&gt;&gt; </span>pd.set_option(<span class="hljs-string">&quot;display.max_colwidth&quot;</span>, <span class="hljs-literal">None</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>generated_questions = pd.DataFrame.from_dict(outputs)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Evaluation dataset before filtering:&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>display(
<span class="hljs-meta">... </span>    generated_questions[
<span class="hljs-meta">... </span>        [
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;question&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;answer&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;groundedness_score&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;relevance_score&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;standalone_score&quot;</span>,
<span class="hljs-meta">... </span>        ]
<span class="hljs-meta">... </span>    ]
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_questions = generated_questions.loc[
<span class="hljs-meta">... </span>    (generated_questions[<span class="hljs-string">&quot;groundedness_score&quot;</span>] &gt;= <span class="hljs-number">4</span>)
<span class="hljs-meta">... </span>    &amp; (generated_questions[<span class="hljs-string">&quot;relevance_score&quot;</span>] &gt;= <span class="hljs-number">4</span>)
<span class="hljs-meta">... </span>    &amp; (generated_questions[<span class="hljs-string">&quot;standalone_score&quot;</span>] &gt;= <span class="hljs-number">4</span>)
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;============================================&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Final evaluation dataset:&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>display(
<span class="hljs-meta">... </span>    generated_questions[
<span class="hljs-meta">... </span>        [
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;question&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;answer&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;groundedness_score&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;relevance_score&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;standalone_score&quot;</span>,
<span class="hljs-meta">... </span>        ]
<span class="hljs-meta">... </span>    ]
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>eval_dataset = datasets.Dataset.from_pandas(generated_questions, split=<span class="hljs-string">&quot;train&quot;</span>, preserve_index=<span class="hljs-literal">False</span>)`,wrap:!1}}),wl=new w({props:{code:"ZXZhbF9kYXRhc2V0JTIwJTNEJTIwZGF0YXNldHMubG9hZF9kYXRhc2V0KCUyMm0tcmljJTJGaHVnZ2luZ2ZhY2VfZG9jX3FhX2V2YWwlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKQ==",highlighted:'eval_dataset = datasets.load_dataset(<span class="hljs-string">&quot;m-ric/huggingface_doc_qa_eval&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)',wrap:!1}}),cl=new c({props:{title:"2. Build our RAG System",local:"2-build-our-rag-system",headingTag:"h1"}}),ul=new c({props:{title:"2.1. Preprocessing documents to build our vector database",local:"21-preprocessing-documents-to-build-our-vector-database",headingTag:"h3"}}),Zl=new w({props:{code:"ZnJvbSUyMGxhbmdjaGFpbi5kb2NzdG9yZS5kb2N1bWVudCUyMGltcG9ydCUyMERvY3VtZW50JTIwYXMlMjBMYW5nY2hhaW5Eb2N1bWVudCUwQSUwQVJBV19LTk9XTEVER0VfQkFTRSUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMExhbmdjaGFpbkRvY3VtZW50KHBhZ2VfY29udGVudCUzRGRvYyU1QiUyMnRleHQlMjIlNUQlMkMlMjBtZXRhZGF0YSUzRCU3QiUyMnNvdXJjZSUyMiUzQSUyMGRvYyU1QiUyMnNvdXJjZSUyMiU1RCU3RCklMjBmb3IlMjBkb2MlMjBpbiUyMHRxZG0oZHMpJTBBJTVE",highlighted:`<span class="hljs-keyword">from</span> langchain.docstore.document <span class="hljs-keyword">import</span> Document <span class="hljs-keyword">as</span> LangchainDocument

RAW_KNOWLEDGE_BASE = [
    LangchainDocument(page_content=doc[<span class="hljs-string">&quot;text&quot;</span>], metadata={<span class="hljs-string">&quot;source&quot;</span>: doc[<span class="hljs-string">&quot;source&quot;</span>]}) <span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> tqdm(ds)
]`,wrap:!1}}),Bl=new w({props:{code:"ZnJvbSUyMGxhbmdjaGFpbi50ZXh0X3NwbGl0dGVyJTIwaW1wb3J0JTIwUmVjdXJzaXZlQ2hhcmFjdGVyVGV4dFNwbGl0dGVyJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEElMEFkZWYlMjBzcGxpdF9kb2N1bWVudHMoJTBBJTIwJTIwJTIwJTIwY2h1bmtfc2l6ZSUzQSUyMGludCUyQyUwQSUyMCUyMCUyMCUyMGtub3dsZWRnZV9iYXNlJTNBJTIwTGlzdCU1QkxhbmdjaGFpbkRvY3VtZW50JTVEJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVyX25hbWUlM0ElMjBzdHIlMkMlMEEpJTIwLSUzRSUyMExpc3QlNUJMYW5nY2hhaW5Eb2N1bWVudCU1RCUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMFNwbGl0JTIwZG9jdW1lbnRzJTIwaW50byUyMGNodW5rcyUyMG9mJTIwc2l6ZSUyMCU2MGNodW5rX3NpemUlNjAlMjBjaGFyYWN0ZXJzJTIwYW5kJTIwcmV0dXJuJTIwYSUyMGxpc3QlMjBvZiUyMGRvY3VtZW50cy4lMEElMjAlMjAlMjAlMjAlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjB0ZXh0X3NwbGl0dGVyJTIwJTNEJTIwUmVjdXJzaXZlQ2hhcmFjdGVyVGV4dFNwbGl0dGVyLmZyb21faHVnZ2luZ2ZhY2VfdG9rZW5pemVyKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKHRva2VuaXplcl9uYW1lKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNodW5rX3NpemUlM0RjaHVua19zaXplJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY2h1bmtfb3ZlcmxhcCUzRGludChjaHVua19zaXplJTIwJTJGJTIwMTApJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWRkX3N0YXJ0X2luZGV4JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0cmlwX3doaXRlc3BhY2UlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VwYXJhdG9ycyUzRCU1QiUyMiU1Q24lNUNuJTIyJTJDJTIwJTIyJTVDbiUyMiUyQyUyMCUyMi4lMjIlMkMlMjAlMjIlMjAlMjIlMkMlMjAlMjIlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjApJTBBJTBBJTIwJTIwJTIwJTIwZG9jc19wcm9jZXNzZWQlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBkb2MlMjBpbiUyMGtub3dsZWRnZV9iYXNlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZG9jc19wcm9jZXNzZWQlMjAlMkIlM0QlMjB0ZXh0X3NwbGl0dGVyLnNwbGl0X2RvY3VtZW50cyglNUJkb2MlNUQpJTBBJTBBJTIwJTIwJTIwJTIwJTIzJTIwUmVtb3ZlJTIwZHVwbGljYXRlcyUwQSUyMCUyMCUyMCUyMHVuaXF1ZV90ZXh0cyUyMCUzRCUyMCU3QiU3RCUwQSUyMCUyMCUyMCUyMGRvY3NfcHJvY2Vzc2VkX3VuaXF1ZSUyMCUzRCUyMCU1QiU1RCUwQSUyMCUyMCUyMCUyMGZvciUyMGRvYyUyMGluJTIwZG9jc19wcm9jZXNzZWQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMGRvYy5wYWdlX2NvbnRlbnQlMjBub3QlMjBpbiUyMHVuaXF1ZV90ZXh0cyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHVuaXF1ZV90ZXh0cyU1QmRvYy5wYWdlX2NvbnRlbnQlNUQlMjAlM0QlMjBUcnVlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZG9jc19wcm9jZXNzZWRfdW5pcXVlLmFwcGVuZChkb2MpJTBBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwZG9jc19wcm9jZXNzZWRfdW5pcXVl",highlighted:`<span class="hljs-keyword">from</span> langchain.text_splitter <span class="hljs-keyword">import</span> RecursiveCharacterTextSplitter
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer


<span class="hljs-keyword">def</span> <span class="hljs-title function_">split_documents</span>(<span class="hljs-params">
    chunk_size: <span class="hljs-built_in">int</span>,
    knowledge_base: <span class="hljs-type">List</span>[LangchainDocument],
    tokenizer_name: <span class="hljs-built_in">str</span>,
</span>) -&gt; <span class="hljs-type">List</span>[LangchainDocument]:
    <span class="hljs-string">&quot;&quot;&quot;
    Split documents into chunks of size \`chunk_size\` characters and return a list of documents.
    &quot;&quot;&quot;</span>
    text_splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer(
        AutoTokenizer.from_pretrained(tokenizer_name),
        chunk_size=chunk_size,
        chunk_overlap=<span class="hljs-built_in">int</span>(chunk_size / <span class="hljs-number">10</span>),
        add_start_index=<span class="hljs-literal">True</span>,
        strip_whitespace=<span class="hljs-literal">True</span>,
        separators=[<span class="hljs-string">&quot;\\n\\n&quot;</span>, <span class="hljs-string">&quot;\\n&quot;</span>, <span class="hljs-string">&quot;.&quot;</span>, <span class="hljs-string">&quot; &quot;</span>, <span class="hljs-string">&quot;&quot;</span>],
    )

    docs_processed = []
    <span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> knowledge_base:
        docs_processed += text_splitter.split_documents([doc])

    <span class="hljs-comment"># Remove duplicates</span>
    unique_texts = {}
    docs_processed_unique = []
    <span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> docs_processed:
        <span class="hljs-keyword">if</span> doc.page_content <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> unique_texts:
            unique_texts[doc.page_content] = <span class="hljs-literal">True</span>
            docs_processed_unique.append(doc)

    <span class="hljs-keyword">return</span> docs_processed_unique`,wrap:!1}}),Gl=new c({props:{title:"2.2. Retriever - embeddings 🗂️",local:"22-retriever---embeddings-",headingTag:"h3"}}),Vl=new w({props:{code:"ZnJvbSUyMGxhbmdjaGFpbi52ZWN0b3JzdG9yZXMlMjBpbXBvcnQlMjBGQUlTUyUwQWZyb20lMjBsYW5nY2hhaW5fY29tbXVuaXR5LmVtYmVkZGluZ3MlMjBpbXBvcnQlMjBIdWdnaW5nRmFjZUVtYmVkZGluZ3MlMEFmcm9tJTIwbGFuZ2NoYWluX2NvbW11bml0eS52ZWN0b3JzdG9yZXMudXRpbHMlMjBpbXBvcnQlMjBEaXN0YW5jZVN0cmF0ZWd5JTBBaW1wb3J0JTIwb3MlMEElMEElMEFkZWYlMjBsb2FkX2VtYmVkZGluZ3MoJTBBJTIwJTIwJTIwJTIwbGFuZ2NoYWluX2RvY3MlM0ElMjBMaXN0JTVCTGFuZ2NoYWluRG9jdW1lbnQlNUQlMkMlMEElMjAlMjAlMjAlMjBjaHVua19zaXplJTNBJTIwaW50JTJDJTBBJTIwJTIwJTIwJTIwZW1iZWRkaW5nX21vZGVsX25hbWUlM0ElMjBPcHRpb25hbCU1QnN0ciU1RCUyMCUzRCUyMCUyMnRoZW5scGVyJTJGZ3RlLXNtYWxsJTIyJTJDJTBBKSUyMC0lM0UlMjBGQUlTUyUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMENyZWF0ZXMlMjBhJTIwRkFJU1MlMjBpbmRleCUyMGZyb20lMjB0aGUlMjBnaXZlbiUyMGVtYmVkZGluZyUyMG1vZGVsJTIwYW5kJTIwZG9jdW1lbnRzLiUyMExvYWRzJTIwdGhlJTIwaW5kZXglMjBkaXJlY3RseSUyMGlmJTIwaXQlMjBhbHJlYWR5JTIwZXhpc3RzLiUwQSUwQSUyMCUyMCUyMCUyMEFyZ3MlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYW5nY2hhaW5fZG9jcyUzQSUyMGxpc3QlMjBvZiUyMGRvY3VtZW50cyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNodW5rX3NpemUlM0ElMjBzaXplJTIwb2YlMjB0aGUlMjBjaHVua3MlMjB0byUyMHNwbGl0JTIwdGhlJTIwZG9jdW1lbnRzJTIwaW50byUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVtYmVkZGluZ19tb2RlbF9uYW1lJTNBJTIwbmFtZSUyMG9mJTIwdGhlJTIwZW1iZWRkaW5nJTIwbW9kZWwlMjB0byUyMHVzZSUwQSUwQSUyMCUyMCUyMCUyMFJldHVybnMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBGQUlTUyUyMGluZGV4JTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwJTIzJTIwbG9hZCUyMGVtYmVkZGluZ19tb2RlbCUwQSUyMCUyMCUyMCUyMGVtYmVkZGluZ19tb2RlbCUyMCUzRCUyMEh1Z2dpbmdGYWNlRW1iZWRkaW5ncyglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtb2RlbF9uYW1lJTNEZW1iZWRkaW5nX21vZGVsX25hbWUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtdWx0aV9wcm9jZXNzJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1vZGVsX2t3YXJncyUzRCU3QiUyMmRldmljZSUyMiUzQSUyMCUyMmN1ZGElMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmNvZGVfa3dhcmdzJTNEJTdCJTIybm9ybWFsaXplX2VtYmVkZGluZ3MlMjIlM0ElMjBUcnVlJTdEJTJDJTIwJTIwJTIzJTIwc2V0JTIwVHJ1ZSUyMHRvJTIwY29tcHV0ZSUyMGNvc2luZSUyMHNpbWlsYXJpdHklMEElMjAlMjAlMjAlMjApJTBBJTBBJTIwJTIwJTIwJTIwJTIzJTIwQ2hlY2slMjBpZiUyMGVtYmVkZGluZ3MlMjBhbHJlYWR5JTIwZXhpc3QlMjBvbiUyMGRpc2slMEElMjAlMjAlMjAlMjBpbmRleF9uYW1lJTIwJTNEJTIwZiUyMmluZGV4X2NodW5rJTNBJTdCY2h1bmtfc2l6ZSU3RF9lbWJlZGRpbmdzJTNBJTdCZW1iZWRkaW5nX21vZGVsX25hbWUucmVwbGFjZSgnJTJGJyUyQyUyMCd+JyklN0QlMjIlMEElMjAlMjAlMjAlMjBpbmRleF9mb2xkZXJfcGF0aCUyMCUzRCUyMGYlMjIuJTJGZGF0YSUyRmluZGV4ZXMlMkYlN0JpbmRleF9uYW1lJTdEJTJGJTIyJTBBJTIwJTIwJTIwJTIwaWYlMjBvcy5wYXRoLmlzZGlyKGluZGV4X2ZvbGRlcl9wYXRoKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMEZBSVNTLmxvYWRfbG9jYWwoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW5kZXhfZm9sZGVyX3BhdGglMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbWJlZGRpbmdfbW9kZWwlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBkaXN0YW5jZV9zdHJhdGVneSUzRERpc3RhbmNlU3RyYXRlZ3kuQ09TSU5FJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUwQSUwQSUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludCglMjJJbmRleCUyMG5vdCUyMGZvdW5kJTJDJTIwZ2VuZXJhdGluZyUyMGl0Li4uJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGRvY3NfcHJvY2Vzc2VkJTIwJTNEJTIwc3BsaXRfZG9jdW1lbnRzKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNodW5rX3NpemUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYW5nY2hhaW5fZG9jcyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVtYmVkZGluZ19tb2RlbF9uYW1lJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGtub3dsZWRnZV9pbmRleCUyMCUzRCUyMEZBSVNTLmZyb21fZG9jdW1lbnRzKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGRvY3NfcHJvY2Vzc2VkJTJDJTIwZW1iZWRkaW5nX21vZGVsJTJDJTIwZGlzdGFuY2Vfc3RyYXRlZ3klM0REaXN0YW5jZVN0cmF0ZWd5LkNPU0lORSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBrbm93bGVkZ2VfaW5kZXguc2F2ZV9sb2NhbChpbmRleF9mb2xkZXJfcGF0aCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjBrbm93bGVkZ2VfaW5kZXg=",highlighted:`<span class="hljs-keyword">from</span> langchain.vectorstores <span class="hljs-keyword">import</span> FAISS
<span class="hljs-keyword">from</span> langchain_community.embeddings <span class="hljs-keyword">import</span> HuggingFaceEmbeddings
<span class="hljs-keyword">from</span> langchain_community.vectorstores.utils <span class="hljs-keyword">import</span> DistanceStrategy
<span class="hljs-keyword">import</span> os


<span class="hljs-keyword">def</span> <span class="hljs-title function_">load_embeddings</span>(<span class="hljs-params">
    langchain_docs: <span class="hljs-type">List</span>[LangchainDocument],
    chunk_size: <span class="hljs-built_in">int</span>,
    embedding_model_name: <span class="hljs-type">Optional</span>[<span class="hljs-built_in">str</span>] = <span class="hljs-string">&quot;thenlper/gte-small&quot;</span>,
</span>) -&gt; FAISS:
    <span class="hljs-string">&quot;&quot;&quot;
    Creates a FAISS index from the given embedding model and documents. Loads the index directly if it already exists.

    Args:
        langchain_docs: list of documents
        chunk_size: size of the chunks to split the documents into
        embedding_model_name: name of the embedding model to use

    Returns:
        FAISS index
    &quot;&quot;&quot;</span>
    <span class="hljs-comment"># load embedding_model</span>
    embedding_model = HuggingFaceEmbeddings(
        model_name=embedding_model_name,
        multi_process=<span class="hljs-literal">True</span>,
        model_kwargs={<span class="hljs-string">&quot;device&quot;</span>: <span class="hljs-string">&quot;cuda&quot;</span>},
        encode_kwargs={<span class="hljs-string">&quot;normalize_embeddings&quot;</span>: <span class="hljs-literal">True</span>},  <span class="hljs-comment"># set True to compute cosine similarity</span>
    )

    <span class="hljs-comment"># Check if embeddings already exist on disk</span>
    index_name = <span class="hljs-string">f&quot;index_chunk:<span class="hljs-subst">{chunk_size}</span>_embeddings:<span class="hljs-subst">{embedding_model_name.replace(<span class="hljs-string">&#x27;/&#x27;</span>, <span class="hljs-string">&#x27;~&#x27;</span>)}</span>&quot;</span>
    index_folder_path = <span class="hljs-string">f&quot;./data/indexes/<span class="hljs-subst">{index_name}</span>/&quot;</span>
    <span class="hljs-keyword">if</span> os.path.isdir(index_folder_path):
        <span class="hljs-keyword">return</span> FAISS.load_local(
            index_folder_path,
            embedding_model,
            distance_strategy=DistanceStrategy.COSINE,
        )

    <span class="hljs-keyword">else</span>:
        <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Index not found, generating it...&quot;</span>)
        docs_processed = split_documents(
            chunk_size,
            langchain_docs,
            embedding_model_name,
        )
        knowledge_index = FAISS.from_documents(
            docs_processed, embedding_model, distance_strategy=DistanceStrategy.COSINE
        )
        knowledge_index.save_local(index_folder_path)
        <span class="hljs-keyword">return</span> knowledge_index`,wrap:!1}}),Ql=new c({props:{title:"2.3. Reader - LLM 💬",local:"23-reader---llm-",headingTag:"h3"}}),Xl=new w({props:{code:"UkFHX1BST01QVF9URU1QTEFURSUyMCUzRCUyMCUyMiUyMiUyMiUwQSUzQyU3Q3N5c3RlbSU3QyUzRSUwQVVzaW5nJTIwdGhlJTIwaW5mb3JtYXRpb24lMjBjb250YWluZWQlMjBpbiUyMHRoZSUyMGNvbnRleHQlMkMlMEFnaXZlJTIwYSUyMGNvbXByZWhlbnNpdmUlMjBhbnN3ZXIlMjB0byUyMHRoZSUyMHF1ZXN0aW9uLiUwQVJlc3BvbmQlMjBvbmx5JTIwdG8lMjB0aGUlMjBxdWVzdGlvbiUyMGFza2VkJTJDJTIwcmVzcG9uc2UlMjBzaG91bGQlMjBiZSUyMGNvbmNpc2UlMjBhbmQlMjByZWxldmFudCUyMHRvJTIwdGhlJTIwcXVlc3Rpb24uJTBBUHJvdmlkZSUyMHRoZSUyMG51bWJlciUyMG9mJTIwdGhlJTIwc291cmNlJTIwZG9jdW1lbnQlMjB3aGVuJTIwcmVsZXZhbnQuJTBBSWYlMjB0aGUlMjBhbnN3ZXIlMjBjYW5ub3QlMjBiZSUyMGRlZHVjZWQlMjBmcm9tJTIwdGhlJTIwY29udGV4dCUyQyUyMGRvJTIwbm90JTIwZ2l2ZSUyMGFuJTIwYW5zd2VyLiUzQyUyRnMlM0UlMEElM0MlN0N1c2VyJTdDJTNFJTBBQ29udGV4dCUzQSUwQSU3QmNvbnRleHQlN0QlMEEtLS0lMEFOb3clMjBoZXJlJTIwaXMlMjB0aGUlMjBxdWVzdGlvbiUyMHlvdSUyMG5lZWQlMjB0byUyMGFuc3dlci4lMEElMEFRdWVzdGlvbiUzQSUyMCU3QnF1ZXN0aW9uJTdEJTBBJTNDJTJGcyUzRSUwQSUzQyU3Q2Fzc2lzdGFudCU3QyUzRSUwQSUyMiUyMiUyMg==",highlighted:`RAG_PROMPT_TEMPLATE = <span class="hljs-string">&quot;&quot;&quot;
&lt;|system|&gt;
Using the information contained in the context,
give a comprehensive answer to the question.
Respond only to the question asked, response should be concise and relevant to the question.
Provide the number of the source document when relevant.
If the answer cannot be deduced from the context, do not give an answer.&lt;/s&gt;
&lt;|user|&gt;
Context:
{context}
---
Now here is the question you need to answer.

Question: {question}
&lt;/s&gt;
&lt;|assistant|&gt;
&quot;&quot;&quot;</span>`,wrap:!1}}),kl=new w({props:{code:"ZnJvbSUyMGxhbmdjaGFpbl9jb21tdW5pdHkubGxtcyUyMGltcG9ydCUyMEh1Z2dpbmdGYWNlSHViJTBBJTBBcmVwb19pZCUyMCUzRCUyMCUyMkh1Z2dpbmdGYWNlSDQlMkZ6ZXBoeXItN2ItYmV0YSUyMiUwQVJFQURFUl9NT0RFTF9OQU1FJTIwJTNEJTIwJTIyemVwaHlyLTdiLWJldGElMjIlMEFIRl9BUElfVE9LRU4lMjAlM0QlMjAlMjIlMjIlMEElMEFSRUFERVJfTExNJTIwJTNEJTIwSHVnZ2luZ0ZhY2VIdWIoJTBBJTIwJTIwJTIwJTIwcmVwb19pZCUzRHJlcG9faWQlMkMlMEElMjAlMjAlMjAlMjB0YXNrJTNEJTIydGV4dC1nZW5lcmF0aW9uJTIyJTJDJTBBJTIwJTIwJTIwJTIwaHVnZ2luZ2ZhY2VodWJfYXBpX3Rva2VuJTNESEZfQVBJX1RPS0VOJTJDJTBBJTIwJTIwJTIwJTIwbW9kZWxfa3dhcmdzJTNEJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybWF4X25ld190b2tlbnMlMjIlM0ElMjA1MTIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0b3BfayUyMiUzQSUyMDMwJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydGVtcGVyYXR1cmUlMjIlM0ElMjAwLjElMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyZXBldGl0aW9uX3BlbmFsdHklMjIlM0ElMjAxLjAzJTJDJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> langchain_community.llms <span class="hljs-keyword">import</span> HuggingFaceHub

repo_id = <span class="hljs-string">&quot;HuggingFaceH4/zephyr-7b-beta&quot;</span>
READER_MODEL_NAME = <span class="hljs-string">&quot;zephyr-7b-beta&quot;</span>
HF_API_TOKEN = <span class="hljs-string">&quot;&quot;</span>

READER_LLM = HuggingFaceHub(
    repo_id=repo_id,
    task=<span class="hljs-string">&quot;text-generation&quot;</span>,
    huggingfacehub_api_token=HF_API_TOKEN,
    model_kwargs={
        <span class="hljs-string">&quot;max_new_tokens&quot;</span>: <span class="hljs-number">512</span>,
        <span class="hljs-string">&quot;top_k&quot;</span>: <span class="hljs-number">30</span>,
        <span class="hljs-string">&quot;temperature&quot;</span>: <span class="hljs-number">0.1</span>,
        <span class="hljs-string">&quot;repetition_penalty&quot;</span>: <span class="hljs-number">1.03</span>,
    },
)`,wrap:!1}}),El=new w({props:{code:"ZnJvbSUyMHJhZ2F0b3VpbGxlJTIwaW1wb3J0JTIwUkFHUHJldHJhaW5lZE1vZGVsJTBBZnJvbSUyMGxhbmdjaGFpbl9jb3JlLnZlY3RvcnN0b3JlcyUyMGltcG9ydCUyMFZlY3RvclN0b3JlJTBBZnJvbSUyMGxhbmdjaGFpbl9jb3JlLmxhbmd1YWdlX21vZGVscy5sbG1zJTIwaW1wb3J0JTIwTExNJTBBJTBBJTBBZGVmJTIwYW5zd2VyX3dpdGhfcmFnKCUwQSUyMCUyMCUyMCUyMHF1ZXN0aW9uJTNBJTIwc3RyJTJDJTBBJTIwJTIwJTIwJTIwbGxtJTNBJTIwTExNJTJDJTBBJTIwJTIwJTIwJTIwa25vd2xlZGdlX2luZGV4JTNBJTIwVmVjdG9yU3RvcmUlMkMlMEElMjAlMjAlMjAlMjByZXJhbmtlciUzQSUyME9wdGlvbmFsJTVCUkFHUHJldHJhaW5lZE1vZGVsJTVEJTIwJTNEJTIwTm9uZSUyQyUwQSUyMCUyMCUyMCUyMG51bV9yZXRyaWV2ZWRfZG9jcyUzQSUyMGludCUyMCUzRCUyMDMwJTJDJTBBJTIwJTIwJTIwJTIwbnVtX2RvY3NfZmluYWwlM0ElMjBpbnQlMjAlM0QlMjA3JTJDJTBBKSUyMC0lM0UlMjBUdXBsZSU1QnN0ciUyQyUyMExpc3QlNUJMYW5nY2hhaW5Eb2N1bWVudCU1RCU1RCUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMkFuc3dlciUyMGElMjBxdWVzdGlvbiUyMHVzaW5nJTIwUkFHJTIwd2l0aCUyMHRoZSUyMGdpdmVuJTIwa25vd2xlZGdlJTIwaW5kZXguJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwJTIzJTIwR2F0aGVyJTIwZG9jdW1lbnRzJTIwd2l0aCUyMHJldHJpZXZlciUwQSUyMCUyMCUyMCUyMHJlbGV2YW50X2RvY3MlMjAlM0QlMjBrbm93bGVkZ2VfaW5kZXguc2ltaWxhcml0eV9zZWFyY2gocXVlcnklM0RxdWVzdGlvbiUyQyUyMGslM0RudW1fcmV0cmlldmVkX2RvY3MpJTBBJTIwJTIwJTIwJTIwcmVsZXZhbnRfZG9jcyUyMCUzRCUyMCU1QmRvYy5wYWdlX2NvbnRlbnQlMjBmb3IlMjBkb2MlMjBpbiUyMHJlbGV2YW50X2RvY3MlNUQlMjAlMjAlMjMlMjBrZWVwJTIwb25seSUyMHRoZSUyMHRleHQlMEElMEElMjAlMjAlMjAlMjAlMjMlMjBPcHRpb25hbGx5JTIwcmVyYW5rJTIwcmVzdWx0cyUwQSUyMCUyMCUyMCUyMGlmJTIwcmVyYW5rZXIlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZWxldmFudF9kb2NzJTIwJTNEJTIwcmVyYW5rZXIucmVyYW5rKHF1ZXN0aW9uJTJDJTIwcmVsZXZhbnRfZG9jcyUyQyUyMGslM0RudW1fZG9jc19maW5hbCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZWxldmFudF9kb2NzJTIwJTNEJTIwJTVCZG9jJTVCJTIyY29udGVudCUyMiU1RCUyMGZvciUyMGRvYyUyMGluJTIwcmVsZXZhbnRfZG9jcyU1RCUwQSUwQSUyMCUyMCUyMCUyMHJlbGV2YW50X2RvY3MlMjAlM0QlMjByZWxldmFudF9kb2NzJTVCJTNBbnVtX2RvY3NfZmluYWwlNUQlMEElMEElMjAlMjAlMjAlMjAlMjMlMjBCdWlsZCUyMHRoZSUyMGZpbmFsJTIwcHJvbXB0JTBBJTIwJTIwJTIwJTIwY29udGV4dCUyMCUzRCUyMCUyMiU1Q25FeHRyYWN0ZWQlMjBkb2N1bWVudHMlM0ElNUNuJTIyJTBBJTIwJTIwJTIwJTIwY29udGV4dCUyMCUyQiUzRCUyMCUyMiUyMi5qb2luKCU1QmYlMjJEb2N1bWVudCUyMCU3QnN0cihpKSU3RCUzQSUzQSUzQSU1Q24lMjIlMjAlMkIlMjBkb2MlMjBmb3IlMjBpJTJDJTIwZG9jJTIwaW4lMjBlbnVtZXJhdGUocmVsZXZhbnRfZG9jcyklNUQpJTBBJTBBJTIwJTIwJTIwJTIwZmluYWxfcHJvbXB0JTIwJTNEJTIwUkFHX1BST01QVF9URU1QTEFURS5mb3JtYXQocXVlc3Rpb24lM0RxdWVzdGlvbiUyQyUyMGNvbnRleHQlM0Rjb250ZXh0KSUwQSUwQSUyMCUyMCUyMCUyMCUyMyUyMFJlZGFjdCUyMGFuJTIwYW5zd2VyJTBBJTIwJTIwJTIwJTIwYW5zd2VyJTIwJTNEJTIwbGxtKGZpbmFsX3Byb21wdCklMEElMEElMjAlMjAlMjAlMjByZXR1cm4lMjBhbnN3ZXIlMkMlMjByZWxldmFudF9kb2Nz",highlighted:`<span class="hljs-keyword">from</span> ragatouille <span class="hljs-keyword">import</span> RAGPretrainedModel
<span class="hljs-keyword">from</span> langchain_core.vectorstores <span class="hljs-keyword">import</span> VectorStore
<span class="hljs-keyword">from</span> langchain_core.language_models.llms <span class="hljs-keyword">import</span> LLM


<span class="hljs-keyword">def</span> <span class="hljs-title function_">answer_with_rag</span>(<span class="hljs-params">
    question: <span class="hljs-built_in">str</span>,
    llm: LLM,
    knowledge_index: VectorStore,
    reranker: <span class="hljs-type">Optional</span>[RAGPretrainedModel] = <span class="hljs-literal">None</span>,
    num_retrieved_docs: <span class="hljs-built_in">int</span> = <span class="hljs-number">30</span>,
    num_docs_final: <span class="hljs-built_in">int</span> = <span class="hljs-number">7</span>,
</span>) -&gt; <span class="hljs-type">Tuple</span>[<span class="hljs-built_in">str</span>, <span class="hljs-type">List</span>[LangchainDocument]]:
    <span class="hljs-string">&quot;&quot;&quot;Answer a question using RAG with the given knowledge index.&quot;&quot;&quot;</span>
    <span class="hljs-comment"># Gather documents with retriever</span>
    relevant_docs = knowledge_index.similarity_search(query=question, k=num_retrieved_docs)
    relevant_docs = [doc.page_content <span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> relevant_docs]  <span class="hljs-comment"># keep only the text</span>

    <span class="hljs-comment"># Optionally rerank results</span>
    <span class="hljs-keyword">if</span> reranker:
        relevant_docs = reranker.rerank(question, relevant_docs, k=num_docs_final)
        relevant_docs = [doc[<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> relevant_docs]

    relevant_docs = relevant_docs[:num_docs_final]

    <span class="hljs-comment"># Build the final prompt</span>
    context = <span class="hljs-string">&quot;\\nExtracted documents:\\n&quot;</span>
    context += <span class="hljs-string">&quot;&quot;</span>.join([<span class="hljs-string">f&quot;Document <span class="hljs-subst">{<span class="hljs-built_in">str</span>(i)}</span>:::\\n&quot;</span> + doc <span class="hljs-keyword">for</span> i, doc <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(relevant_docs)])

    final_prompt = RAG_PROMPT_TEMPLATE.<span class="hljs-built_in">format</span>(question=question, context=context)

    <span class="hljs-comment"># Redact an answer</span>
    answer = llm(final_prompt)

    <span class="hljs-keyword">return</span> answer, relevant_docs`,wrap:!1}}),Sl=new c({props:{title:"3. Benchmarking the RAG system",local:"3-benchmarking-the-rag-system",headingTag:"h1"}}),Hl=new w({props:{code:"ZnJvbSUyMGxhbmdjaGFpbl9jb3JlLmxhbmd1YWdlX21vZGVscyUyMGltcG9ydCUyMEJhc2VDaGF0TW9kZWwlMEElMEElMEFkZWYlMjBydW5fcmFnX3Rlc3RzKCUwQSUyMCUyMCUyMCUyMGV2YWxfZGF0YXNldCUzQSUyMGRhdGFzZXRzLkRhdGFzZXQlMkMlMEElMjAlMjAlMjAlMjBsbG0lM0ElMjBCYXNlQ2hhdE1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwa25vd2xlZGdlX2luZGV4JTNBJTIwVmVjdG9yU3RvcmUlMkMlMEElMjAlMjAlMjAlMjBvdXRwdXRfZmlsZSUzQSUyMHN0ciUyQyUwQSUyMCUyMCUyMCUyMHJlcmFua2VyJTNBJTIwT3B0aW9uYWwlNUJSQUdQcmV0cmFpbmVkTW9kZWwlNUQlMjAlM0QlMjBOb25lJTJDJTBBJTIwJTIwJTIwJTIwdmVyYm9zZSUzQSUyME9wdGlvbmFsJTVCYm9vbCU1RCUyMCUzRCUyMFRydWUlMkMlMEElMjAlMjAlMjAlMjB0ZXN0X3NldHRpbmdzJTNBJTIwT3B0aW9uYWwlNUJzdHIlNUQlMjAlM0QlMjBOb25lJTJDJTIwJTIwJTIzJTIwVG8lMjBkb2N1bWVudCUyMHRoZSUyMHRlc3QlMjBzZXR0aW5ncyUyMHVzZWQlMEEpJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyUnVucyUyMFJBRyUyMHRlc3RzJTIwb24lMjB0aGUlMjBnaXZlbiUyMGRhdGFzZXQlMjBhbmQlMjBzYXZlcyUyMHRoZSUyMHJlc3VsdHMlMjB0byUyMHRoZSUyMGdpdmVuJTIwb3V0cHV0JTIwZmlsZS4lMjIlMjIlMjIlMEElMjAlMjAlMjAlMjB0cnklM0ElMjAlMjAlMjMlMjBsb2FkJTIwcHJldmlvdXMlMjBnZW5lcmF0aW9ucyUyMGlmJTIwdGhleSUyMGV4aXN0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2l0aCUyMG9wZW4ob3V0cHV0X2ZpbGUlMkMlMjAlMjJyJTIyKSUyMGFzJTIwZiUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjBqc29uLmxvYWQoZiklMEElMjAlMjAlMjAlMjBleGNlcHQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwJTVCJTVEJTBBJTBBJTIwJTIwJTIwJTIwZm9yJTIwZXhhbXBsZSUyMGluJTIwdHFkbShldmFsX2RhdGFzZXQpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcXVlc3Rpb24lMjAlM0QlMjBleGFtcGxlJTVCJTIycXVlc3Rpb24lMjIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMHF1ZXN0aW9uJTIwaW4lMjAlNUJvdXRwdXQlNUIlMjJxdWVzdGlvbiUyMiU1RCUyMGZvciUyMG91dHB1dCUyMGluJTIwb3V0cHV0cyU1RCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNvbnRpbnVlJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYW5zd2VyJTJDJTIwcmVsZXZhbnRfZG9jcyUyMCUzRCUyMGFuc3dlcl93aXRoX3JhZyhxdWVzdGlvbiUyQyUyMGxsbSUyQyUyMGtub3dsZWRnZV9pbmRleCUyQyUyMHJlcmFua2VyJTNEcmVyYW5rZXIpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjB2ZXJib3NlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoJTIyJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTNEJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KGYlMjJRdWVzdGlvbiUzQSUyMCU3QnF1ZXN0aW9uJTdEJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KGYlMjJBbnN3ZXIlM0ElMjAlN0JhbnN3ZXIlN0QlMjIpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoZidUcnVlJTIwYW5zd2VyJTNBJTIwJTdCZXhhbXBsZSU1QiUyMmFuc3dlciUyMiU1RCU3RCcpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0JTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycXVlc3Rpb24lMjIlM0ElMjBxdWVzdGlvbiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnRydWVfYW5zd2VyJTIyJTNBJTIwZXhhbXBsZSU1QiUyMmFuc3dlciUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnNvdXJjZV9kb2MlMjIlM0ElMjBleGFtcGxlJTVCJTIyc291cmNlX2RvYyUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmdlbmVyYXRlZF9hbnN3ZXIlMjIlM0ElMjBhbnN3ZXIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyZXRyaWV2ZWRfZG9jcyUyMiUzQSUyMCU1QmRvYyUyMGZvciUyMGRvYyUyMGluJTIwcmVsZXZhbnRfZG9jcyU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwdGVzdF9zZXR0aW5ncyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJlc3VsdCU1QiUyMnRlc3Rfc2V0dGluZ3MlMjIlNUQlMjAlM0QlMjB0ZXN0X3NldHRpbmdzJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0cy5hcHBlbmQocmVzdWx0KSUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHdpdGglMjBvcGVuKG91dHB1dF9maWxlJTJDJTIwJTIydyUyMiklMjBhcyUyMGYlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBqc29uLmR1bXAob3V0cHV0cyUyQyUyMGYp",highlighted:`<span class="hljs-keyword">from</span> langchain_core.language_models <span class="hljs-keyword">import</span> BaseChatModel


<span class="hljs-keyword">def</span> <span class="hljs-title function_">run_rag_tests</span>(<span class="hljs-params">
    eval_dataset: datasets.Dataset,
    llm: BaseChatModel,
    knowledge_index: VectorStore,
    output_file: <span class="hljs-built_in">str</span>,
    reranker: <span class="hljs-type">Optional</span>[RAGPretrainedModel] = <span class="hljs-literal">None</span>,
    verbose: <span class="hljs-type">Optional</span>[<span class="hljs-built_in">bool</span>] = <span class="hljs-literal">True</span>,
    test_settings: <span class="hljs-type">Optional</span>[<span class="hljs-built_in">str</span>] = <span class="hljs-literal">None</span>,  <span class="hljs-comment"># To document the test settings used</span>
</span>):
    <span class="hljs-string">&quot;&quot;&quot;Runs RAG tests on the given dataset and saves the results to the given output file.&quot;&quot;&quot;</span>
    <span class="hljs-keyword">try</span>:  <span class="hljs-comment"># load previous generations if they exist</span>
        <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(output_file, <span class="hljs-string">&quot;r&quot;</span>) <span class="hljs-keyword">as</span> f:
            outputs = json.load(f)
    <span class="hljs-keyword">except</span>:
        outputs = []

    <span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> tqdm(eval_dataset):
        question = example[<span class="hljs-string">&quot;question&quot;</span>]
        <span class="hljs-keyword">if</span> question <span class="hljs-keyword">in</span> [output[<span class="hljs-string">&quot;question&quot;</span>] <span class="hljs-keyword">for</span> output <span class="hljs-keyword">in</span> outputs]:
            <span class="hljs-keyword">continue</span>

        answer, relevant_docs = answer_with_rag(question, llm, knowledge_index, reranker=reranker)
        <span class="hljs-keyword">if</span> verbose:
            <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;=======================================================&quot;</span>)
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Question: <span class="hljs-subst">{question}</span>&quot;</span>)
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Answer: <span class="hljs-subst">{answer}</span>&quot;</span>)
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&#x27;True answer: <span class="hljs-subst">{example[<span class="hljs-string">&quot;answer&quot;</span>]}</span>&#x27;</span>)
        result = {
            <span class="hljs-string">&quot;question&quot;</span>: question,
            <span class="hljs-string">&quot;true_answer&quot;</span>: example[<span class="hljs-string">&quot;answer&quot;</span>],
            <span class="hljs-string">&quot;source_doc&quot;</span>: example[<span class="hljs-string">&quot;source_doc&quot;</span>],
            <span class="hljs-string">&quot;generated_answer&quot;</span>: answer,
            <span class="hljs-string">&quot;retrieved_docs&quot;</span>: [doc <span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> relevant_docs],
        }
        <span class="hljs-keyword">if</span> test_settings:
            result[<span class="hljs-string">&quot;test_settings&quot;</span>] = test_settings
        outputs.append(result)

        <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(output_file, <span class="hljs-string">&quot;w&quot;</span>) <span class="hljs-keyword">as</span> f:
            json.dump(outputs, f)`,wrap:!1}}),$l=new w({props:{code:"RVZBTFVBVElPTl9QUk9NUFQlMjAlM0QlMjAlMjIlMjIlMjIlMjMlMjMlMjNUYXNrJTIwRGVzY3JpcHRpb24lM0ElMEFBbiUyMGluc3RydWN0aW9uJTIwKG1pZ2h0JTIwaW5jbHVkZSUyMGFuJTIwSW5wdXQlMjBpbnNpZGUlMjBpdCklMkMlMjBhJTIwcmVzcG9uc2UlMjB0byUyMGV2YWx1YXRlJTJDJTIwYSUyMHJlZmVyZW5jZSUyMGFuc3dlciUyMHRoYXQlMjBnZXRzJTIwYSUyMHNjb3JlJTIwb2YlMjA1JTJDJTIwYW5kJTIwYSUyMHNjb3JlJTIwcnVicmljJTIwcmVwcmVzZW50aW5nJTIwYSUyMGV2YWx1YXRpb24lMjBjcml0ZXJpYSUyMGFyZSUyMGdpdmVuLiUwQTEuJTIwV3JpdGUlMjBhJTIwZGV0YWlsZWQlMjBmZWVkYmFjayUyMHRoYXQlMjBhc3Nlc3MlMjB0aGUlMjBxdWFsaXR5JTIwb2YlMjB0aGUlMjByZXNwb25zZSUyMHN0cmljdGx5JTIwYmFzZWQlMjBvbiUyMHRoZSUyMGdpdmVuJTIwc2NvcmUlMjBydWJyaWMlMkMlMjBub3QlMjBldmFsdWF0aW5nJTIwaW4lMjBnZW5lcmFsLiUwQTIuJTIwQWZ0ZXIlMjB3cml0aW5nJTIwYSUyMGZlZWRiYWNrJTJDJTIwd3JpdGUlMjBhJTIwc2NvcmUlMjB0aGF0JTIwaXMlMjBhbiUyMGludGVnZXIlMjBiZXR3ZWVuJTIwMSUyMGFuZCUyMDUuJTIwWW91JTIwc2hvdWxkJTIwcmVmZXIlMjB0byUyMHRoZSUyMHNjb3JlJTIwcnVicmljLiUwQTMuJTIwVGhlJTIwb3V0cHV0JTIwZm9ybWF0JTIwc2hvdWxkJTIwbG9vayUyMGFzJTIwZm9sbG93cyUzQSUyMCU1QyUyMkZlZWRiYWNrJTNBJTIwJTdCJTdCd3JpdGUlMjBhJTIwZmVlZGJhY2slMjBmb3IlMjBjcml0ZXJpYSU3RCU3RCUyMCU1QlJFU1VMVCU1RCUyMCU3QiU3QmFuJTIwaW50ZWdlciUyMG51bWJlciUyMGJldHdlZW4lMjAxJTIwYW5kJTIwNSU3RCU3RCU1QyUyMiUwQTQuJTIwUGxlYXNlJTIwZG8lMjBub3QlMjBnZW5lcmF0ZSUyMGFueSUyMG90aGVyJTIwb3BlbmluZyUyQyUyMGNsb3NpbmclMkMlMjBhbmQlMjBleHBsYW5hdGlvbnMuJTIwQmUlMjBzdXJlJTIwdG8lMjBpbmNsdWRlJTIwJTVCUkVTVUxUJTVEJTIwaW4lMjB5b3VyJTIwb3V0cHV0LiUwQSUwQSUyMyUyMyUyM1RoZSUyMGluc3RydWN0aW9uJTIwdG8lMjBldmFsdWF0ZSUzQSUwQSU3Qmluc3RydWN0aW9uJTdEJTBBJTBBJTIzJTIzJTIzUmVzcG9uc2UlMjB0byUyMGV2YWx1YXRlJTNBJTBBJTdCcmVzcG9uc2UlN0QlMEElMEElMjMlMjMlMjNSZWZlcmVuY2UlMjBBbnN3ZXIlMjAoU2NvcmUlMjA1KSUzQSUwQSU3QnJlZmVyZW5jZV9hbnN3ZXIlN0QlMEElMEElMjMlMjMlMjNTY29yZSUyMFJ1YnJpY3MlM0ElMEElNUJJcyUyMHRoZSUyMHJlc3BvbnNlJTIwY29ycmVjdCUyQyUyMGFjY3VyYXRlJTJDJTIwYW5kJTIwZmFjdHVhbCUyMGJhc2VkJTIwb24lMjB0aGUlMjByZWZlcmVuY2UlMjBhbnN3ZXIlM0YlNUQlMEFTY29yZSUyMDElM0ElMjBUaGUlMjByZXNwb25zZSUyMGlzJTIwY29tcGxldGVseSUyMGluY29ycmVjdCUyQyUyMGluYWNjdXJhdGUlMkMlMjBhbmQlMkZvciUyMG5vdCUyMGZhY3R1YWwuJTBBU2NvcmUlMjAyJTNBJTIwVGhlJTIwcmVzcG9uc2UlMjBpcyUyMG1vc3RseSUyMGluY29ycmVjdCUyQyUyMGluYWNjdXJhdGUlMkMlMjBhbmQlMkZvciUyMG5vdCUyMGZhY3R1YWwuJTBBU2NvcmUlMjAzJTNBJTIwVGhlJTIwcmVzcG9uc2UlMjBpcyUyMHNvbWV3aGF0JTIwY29ycmVjdCUyQyUyMGFjY3VyYXRlJTJDJTIwYW5kJTJGb3IlMjBmYWN0dWFsLiUwQVNjb3JlJTIwNCUzQSUyMFRoZSUyMHJlc3BvbnNlJTIwaXMlMjBtb3N0bHklMjBjb3JyZWN0JTJDJTIwYWNjdXJhdGUlMkMlMjBhbmQlMjBmYWN0dWFsLiUwQVNjb3JlJTIwNSUzQSUyMFRoZSUyMHJlc3BvbnNlJTIwaXMlMjBjb21wbGV0ZWx5JTIwY29ycmVjdCUyQyUyMGFjY3VyYXRlJTJDJTIwYW5kJTIwZmFjdHVhbC4lMEElMEElMjMlMjMlMjNGZWVkYmFjayUzQSUyMiUyMiUyMiUwQSUwQWZyb20lMjBsYW5nY2hhaW4ucHJvbXB0cy5jaGF0JTIwaW1wb3J0JTIwKCUwQSUyMCUyMCUyMCUyMENoYXRQcm9tcHRUZW1wbGF0ZSUyQyUwQSUyMCUyMCUyMCUyMEh1bWFuTWVzc2FnZVByb21wdFRlbXBsYXRlJTJDJTBBKSUwQWZyb20lMjBsYW5nY2hhaW4uc2NoZW1hJTIwaW1wb3J0JTIwU3lzdGVtTWVzc2FnZSUwQSUwQSUwQWV2YWx1YXRpb25fcHJvbXB0X3RlbXBsYXRlJTIwJTNEJTIwQ2hhdFByb21wdFRlbXBsYXRlLmZyb21fbWVzc2FnZXMoJTBBJTIwJTIwJTIwJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwU3lzdGVtTWVzc2FnZShjb250ZW50JTNEJTIyWW91JTIwYXJlJTIwYSUyMGZhaXIlMjBldmFsdWF0b3IlMjBsYW5ndWFnZSUyMG1vZGVsLiUyMiklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBIdW1hbk1lc3NhZ2VQcm9tcHRUZW1wbGF0ZS5mcm9tX3RlbXBsYXRlKEVWQUxVQVRJT05fUFJPTVBUKSUyQyUwQSUyMCUyMCUyMCUyMCU1RCUwQSk=",highlighted:`EVALUATION_PROMPT = <span class="hljs-string">&quot;&quot;&quot;###Task Description:
An instruction (might include an Input inside it), a response to evaluate, a reference answer that gets a score of 5, and a score rubric representing a evaluation criteria are given.
1. Write a detailed feedback that assess the quality of the response strictly based on the given score rubric, not evaluating in general.
2. After writing a feedback, write a score that is an integer between 1 and 5. You should refer to the score rubric.
3. The output format should look as follows: \\&quot;Feedback: {{write a feedback for criteria}} [RESULT] {{an integer number between 1 and 5}}\\&quot;
4. Please do not generate any other opening, closing, and explanations. Be sure to include [RESULT] in your output.

###The instruction to evaluate:
{instruction}

###Response to evaluate:
{response}

###Reference Answer (Score 5):
{reference_answer}

###Score Rubrics:
[Is the response correct, accurate, and factual based on the reference answer?]
Score 1: The response is completely incorrect, inaccurate, and/or not factual.
Score 2: The response is mostly incorrect, inaccurate, and/or not factual.
Score 3: The response is somewhat correct, accurate, and/or factual.
Score 4: The response is mostly correct, accurate, and factual.
Score 5: The response is completely correct, accurate, and factual.

###Feedback:&quot;&quot;&quot;</span>

<span class="hljs-keyword">from</span> langchain.prompts.chat <span class="hljs-keyword">import</span> (
    ChatPromptTemplate,
    HumanMessagePromptTemplate,
)
<span class="hljs-keyword">from</span> langchain.schema <span class="hljs-keyword">import</span> SystemMessage


evaluation_prompt_template = ChatPromptTemplate.from_messages(
    [
        SystemMessage(content=<span class="hljs-string">&quot;You are a fair evaluator language model.&quot;</span>),
        HumanMessagePromptTemplate.from_template(EVALUATION_PROMPT),
    ]
)`,wrap:!1}}),Ll=new w({props:{code:"ZnJvbSUyMGxhbmdjaGFpbi5jaGF0X21vZGVscyUyMGltcG9ydCUyMENoYXRPcGVuQUklMEElMEFPUEVOQUlfQVBJX0tFWSUyMCUzRCUyMCUyMiUyMiUwQSUwQWV2YWxfY2hhdF9tb2RlbCUyMCUzRCUyMENoYXRPcGVuQUkobW9kZWwlM0QlMjJncHQtNC0xMTA2LXByZXZpZXclMjIlMkMlMjB0ZW1wZXJhdHVyZSUzRDAlMkMlMjBvcGVuYWlfYXBpX2tleSUzRE9QRU5BSV9BUElfS0VZKSUwQWV2YWx1YXRvcl9uYW1lJTIwJTNEJTIwJTIyR1BUNCUyMiUwQSUwQSUwQWRlZiUyMGV2YWx1YXRlX2Fuc3dlcnMoJTBBJTIwJTIwJTIwJTIwYW5zd2VyX3BhdGglM0ElMjBzdHIlMkMlMEElMjAlMjAlMjAlMjBldmFsX2NoYXRfbW9kZWwlM0ElMjBCYXNlQ2hhdE1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwZXZhbHVhdG9yX25hbWUlM0ElMjBzdHIlMkMlMEElMjAlMjAlMjAlMjBldmFsdWF0aW9uX3Byb21wdF90ZW1wbGF0ZSUzQSUyMENoYXRQcm9tcHRUZW1wbGF0ZSUyQyUwQSklMjAtJTNFJTIwTm9uZSUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMkV2YWx1YXRlcyUyMGdlbmVyYXRlZCUyMGFuc3dlcnMuJTIwTW9kaWZpZXMlMjB0aGUlMjBnaXZlbiUyMGFuc3dlciUyMGZpbGUlMjBpbiUyMHBsYWNlJTIwZm9yJTIwYmV0dGVyJTIwY2hlY2twb2ludGluZy4lMjIlMjIlMjIlMEElMjAlMjAlMjAlMjBhbnN3ZXJzJTIwJTNEJTIwJTVCJTVEJTBBJTIwJTIwJTIwJTIwaWYlMjBvcy5wYXRoLmlzZmlsZShhbnN3ZXJfcGF0aCklM0ElMjAlMjAlMjMlMjBsb2FkJTIwcHJldmlvdXMlMjBnZW5lcmF0aW9ucyUyMGlmJTIwdGhleSUyMGV4aXN0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYW5zd2VycyUyMCUzRCUyMGpzb24ubG9hZChvcGVuKGFuc3dlcl9wYXRoJTJDJTIwJTIyciUyMikpJTBBJTBBJTIwJTIwJTIwJTIwZm9yJTIwZXhwZXJpbWVudCUyMGluJTIwdHFkbShhbnN3ZXJzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwZiUyMmV2YWxfc2NvcmVfJTdCZXZhbHVhdG9yX25hbWUlN0QlMjIlMjBpbiUyMGV4cGVyaW1lbnQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb250aW51ZSUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGV2YWxfcHJvbXB0JTIwJTNEJTIwZXZhbHVhdGlvbl9wcm9tcHRfdGVtcGxhdGUuZm9ybWF0X21lc3NhZ2VzKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGluc3RydWN0aW9uJTNEZXhwZXJpbWVudCU1QiUyMnF1ZXN0aW9uJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzcG9uc2UlM0RleHBlcmltZW50JTVCJTIyZ2VuZXJhdGVkX2Fuc3dlciUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJlZmVyZW5jZV9hbnN3ZXIlM0RleHBlcmltZW50JTVCJTIydHJ1ZV9hbnN3ZXIlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXZhbF9yZXN1bHQlMjAlM0QlMjBldmFsX2NoYXRfbW9kZWwuaW52b2tlKGV2YWxfcHJvbXB0KSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZlZWRiYWNrJTJDJTIwc2NvcmUlMjAlM0QlMjAlNUJpdGVtLnN0cmlwKCklMjBmb3IlMjBpdGVtJTIwaW4lMjBldmFsX3Jlc3VsdC5jb250ZW50LnNwbGl0KCUyMiU1QlJFU1VMVCU1RCUyMiklNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBleHBlcmltZW50JTVCZiUyMmV2YWxfc2NvcmVfJTdCZXZhbHVhdG9yX25hbWUlN0QlMjIlNUQlMjAlM0QlMjBzY29yZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGV4cGVyaW1lbnQlNUJmJTIyZXZhbF9mZWVkYmFja18lN0JldmFsdWF0b3JfbmFtZSU3RCUyMiU1RCUyMCUzRCUyMGZlZWRiYWNrJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2l0aCUyMG9wZW4oYW5zd2VyX3BhdGglMkMlMjAlMjJ3JTIyKSUyMGFzJTIwZiUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGpzb24uZHVtcChhbnN3ZXJzJTJDJTIwZik=",highlighted:`<span class="hljs-keyword">from</span> langchain.chat_models <span class="hljs-keyword">import</span> ChatOpenAI

OPENAI_API_KEY = <span class="hljs-string">&quot;&quot;</span>

eval_chat_model = ChatOpenAI(model=<span class="hljs-string">&quot;gpt-4-1106-preview&quot;</span>, temperature=<span class="hljs-number">0</span>, openai_api_key=OPENAI_API_KEY)
evaluator_name = <span class="hljs-string">&quot;GPT4&quot;</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">evaluate_answers</span>(<span class="hljs-params">
    answer_path: <span class="hljs-built_in">str</span>,
    eval_chat_model: BaseChatModel,
    evaluator_name: <span class="hljs-built_in">str</span>,
    evaluation_prompt_template: ChatPromptTemplate,
</span>) -&gt; <span class="hljs-literal">None</span>:
    <span class="hljs-string">&quot;&quot;&quot;Evaluates generated answers. Modifies the given answer file in place for better checkpointing.&quot;&quot;&quot;</span>
    answers = []
    <span class="hljs-keyword">if</span> os.path.isfile(answer_path):  <span class="hljs-comment"># load previous generations if they exist</span>
        answers = json.load(<span class="hljs-built_in">open</span>(answer_path, <span class="hljs-string">&quot;r&quot;</span>))

    <span class="hljs-keyword">for</span> experiment <span class="hljs-keyword">in</span> tqdm(answers):
        <span class="hljs-keyword">if</span> <span class="hljs-string">f&quot;eval_score_<span class="hljs-subst">{evaluator_name}</span>&quot;</span> <span class="hljs-keyword">in</span> experiment:
            <span class="hljs-keyword">continue</span>

        eval_prompt = evaluation_prompt_template.format_messages(
            instruction=experiment[<span class="hljs-string">&quot;question&quot;</span>],
            response=experiment[<span class="hljs-string">&quot;generated_answer&quot;</span>],
            reference_answer=experiment[<span class="hljs-string">&quot;true_answer&quot;</span>],
        )
        eval_result = eval_chat_model.invoke(eval_prompt)
        feedback, score = [item.strip() <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> eval_result.content.split(<span class="hljs-string">&quot;[RESULT]&quot;</span>)]
        experiment[<span class="hljs-string">f&quot;eval_score_<span class="hljs-subst">{evaluator_name}</span>&quot;</span>] = score
        experiment[<span class="hljs-string">f&quot;eval_feedback_<span class="hljs-subst">{evaluator_name}</span>&quot;</span>] = feedback

        <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(answer_path, <span class="hljs-string">&quot;w&quot;</span>) <span class="hljs-keyword">as</span> f:
            json.dump(answers, f)`,wrap:!1}}),Kl=new w({props:{code:"aWYlMjBub3QlMjBvcy5wYXRoLmV4aXN0cyglMjIuJTJGb3V0cHV0JTIyKSUzQSUwQSUyMCUyMCUyMCUyMG9zLm1rZGlyKCUyMi4lMkZvdXRwdXQlMjIpJTBBJTBBZm9yJTIwY2h1bmtfc2l6ZSUyMGluJTIwJTVCMjAwJTVEJTNBJTIwJTIwJTIzJTIwQWRkJTIwb3RoZXIlMjBjaHVuayUyMHNpemVzJTIwKGluJTIwdG9rZW5zKSUyMGFzJTIwbmVlZGVkJTBBJTIwJTIwJTIwJTIwZm9yJTIwZW1iZWRkaW5ncyUyMGluJTIwJTVCJTIydGhlbmxwZXIlMkZndGUtc21hbGwlMjIlNUQlM0ElMjAlMjAlMjMlMjBBZGQlMjBvdGhlciUyMGVtYmVkZGluZ3MlMjBhcyUyMG5lZWRlZCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZvciUyMHJlcmFuayUyMGluJTIwJTVCVHJ1ZSUyQyUyMEZhbHNlJTVEJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2V0dGluZ3NfbmFtZSUyMCUzRCUyMGYlMjJjaHVuayUzQSU3QmNodW5rX3NpemUlN0RfZW1iZWRkaW5ncyUzQSU3QmVtYmVkZGluZ3MucmVwbGFjZSgnJTJGJyUyQyUyMCd+JyklN0RfcmVyYW5rJTNBJTdCcmVyYW5rJTdEX3JlYWRlci1tb2RlbCUzQSU3QlJFQURFUl9NT0RFTF9OQU1FJTdEJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0X2ZpbGVfbmFtZSUyMCUzRCUyMGYlMjIuJTJGb3V0cHV0JTJGcmFnXyU3QnNldHRpbmdzX25hbWUlN0QuanNvbiUyMiUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KGYlMjJSdW5uaW5nJTIwZXZhbHVhdGlvbiUyMGZvciUyMCU3QnNldHRpbmdzX25hbWUlN0QlM0ElMjIpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoJTIyTG9hZGluZyUyMGtub3dsZWRnZSUyMGJhc2UlMjBlbWJlZGRpbmdzLi4uJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGtub3dsZWRnZV9pbmRleCUyMCUzRCUyMGxvYWRfZW1iZWRkaW5ncyglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBSQVdfS05PV0xFREdFX0JBU0UlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjaHVua19zaXplJTNEY2h1bmtfc2l6ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVtYmVkZGluZ19tb2RlbF9uYW1lJTNEZW1iZWRkaW5ncyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludCglMjJSdW5uaW5nJTIwUkFHLi4uJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJlcmFua2VyJTIwJTNEJTIwUkFHUHJldHJhaW5lZE1vZGVsLmZyb21fcHJldHJhaW5lZCglMjJjb2xiZXJ0LWlyJTJGY29sYmVydHYyLjAlMjIpJTIwaWYlMjByZXJhbmslMjBlbHNlJTIwTm9uZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJ1bl9yYWdfdGVzdHMoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEZXZhbF9kYXRhc2V0JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGxtJTNEUkVBREVSX0xMTSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGtub3dsZWRnZV9pbmRleCUzRGtub3dsZWRnZV9pbmRleCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG91dHB1dF9maWxlJTNEb3V0cHV0X2ZpbGVfbmFtZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJlcmFua2VyJTNEcmVyYW5rZXIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB2ZXJib3NlJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0ZXN0X3NldHRpbmdzJTNEc2V0dGluZ3NfbmFtZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludCglMjJSdW5uaW5nJTIwZXZhbHVhdGlvbi4uLiUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBldmFsdWF0ZV9hbnN3ZXJzKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG91dHB1dF9maWxlX25hbWUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBldmFsX2NoYXRfbW9kZWwlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBldmFsdWF0b3JfbmFtZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGV2YWx1YXRpb25fcHJvbXB0X3RlbXBsYXRlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKQ==",highlighted:`<span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> os.path.exists(<span class="hljs-string">&quot;./output&quot;</span>):
    os.mkdir(<span class="hljs-string">&quot;./output&quot;</span>)

<span class="hljs-keyword">for</span> chunk_size <span class="hljs-keyword">in</span> [<span class="hljs-number">200</span>]:  <span class="hljs-comment"># Add other chunk sizes (in tokens) as needed</span>
    <span class="hljs-keyword">for</span> embeddings <span class="hljs-keyword">in</span> [<span class="hljs-string">&quot;thenlper/gte-small&quot;</span>]:  <span class="hljs-comment"># Add other embeddings as needed</span>
        <span class="hljs-keyword">for</span> rerank <span class="hljs-keyword">in</span> [<span class="hljs-literal">True</span>, <span class="hljs-literal">False</span>]:
            settings_name = <span class="hljs-string">f&quot;chunk:<span class="hljs-subst">{chunk_size}</span>_embeddings:<span class="hljs-subst">{embeddings.replace(<span class="hljs-string">&#x27;/&#x27;</span>, <span class="hljs-string">&#x27;~&#x27;</span>)}</span>_rerank:<span class="hljs-subst">{rerank}</span>_reader-model:<span class="hljs-subst">{READER_MODEL_NAME}</span>&quot;</span>
            output_file_name = <span class="hljs-string">f&quot;./output/rag_<span class="hljs-subst">{settings_name}</span>.json&quot;</span>

            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Running evaluation for <span class="hljs-subst">{settings_name}</span>:&quot;</span>)

            <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Loading knowledge base embeddings...&quot;</span>)
            knowledge_index = load_embeddings(
                RAW_KNOWLEDGE_BASE,
                chunk_size=chunk_size,
                embedding_model_name=embeddings,
            )

            <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Running RAG...&quot;</span>)
            reranker = RAGPretrainedModel.from_pretrained(<span class="hljs-string">&quot;colbert-ir/colbertv2.0&quot;</span>) <span class="hljs-keyword">if</span> rerank <span class="hljs-keyword">else</span> <span class="hljs-literal">None</span>
            run_rag_tests(
                eval_dataset=eval_dataset,
                llm=READER_LLM,
                knowledge_index=knowledge_index,
                output_file=output_file_name,
                reranker=reranker,
                verbose=<span class="hljs-literal">False</span>,
                test_settings=settings_name,
            )

            <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Running evaluation...&quot;</span>)
            evaluate_answers(
                output_file_name,
                eval_chat_model,
                evaluator_name,
                evaluation_prompt_template,
            )`,wrap:!1}}),Pl=new c({props:{title:"Inspect results",local:"inspect-results",headingTag:"h3"}}),Ol=new w({props:{code:"aW1wb3J0JTIwZ2xvYiUwQSUwQW91dHB1dHMlMjAlM0QlMjAlNUIlNUQlMEFmb3IlMjBmaWxlJTIwaW4lMjBnbG9iLmdsb2IoJTIyLiUyRm91dHB1dCUyRiouanNvbiUyMiklM0ElMEElMjAlMjAlMjAlMjBvdXRwdXQlMjAlM0QlMjBwZC5EYXRhRnJhbWUoanNvbi5sb2FkKG9wZW4oZmlsZSUyQyUyMCUyMnIlMjIpKSklMEElMjAlMjAlMjAlMjBvdXRwdXQlNUIlMjJzZXR0aW5ncyUyMiU1RCUyMCUzRCUyMGZpbGUlMEElMjAlMjAlMjAlMjBvdXRwdXRzLmFwcGVuZChvdXRwdXQpJTBBcmVzdWx0JTIwJTNEJTIwcGQuY29uY2F0KG91dHB1dHMp",highlighted:`<span class="hljs-keyword">import</span> glob

outputs = []
<span class="hljs-keyword">for</span> file <span class="hljs-keyword">in</span> glob.glob(<span class="hljs-string">&quot;./output/*.json&quot;</span>):
    output = pd.DataFrame(json.load(<span class="hljs-built_in">open</span>(file, <span class="hljs-string">&quot;r&quot;</span>)))
    output[<span class="hljs-string">&quot;settings&quot;</span>] = file
    outputs.append(output)
result = pd.concat(outputs)`,wrap:!1}}),ls=new w({props:{code:"cmVzdWx0JTVCJTIyZXZhbF9zY29yZV9HUFQ0JTIyJTVEJTIwJTNEJTIwcmVzdWx0JTVCJTIyZXZhbF9zY29yZV9HUFQ0JTIyJTVELmFwcGx5KGxhbWJkYSUyMHglM0ElMjBpbnQoeCklMjBpZiUyMGlzaW5zdGFuY2UoeCUyQyUyMHN0ciklMjBlbHNlJTIwMSklMEFyZXN1bHQlNUIlMjJldmFsX3Njb3JlX0dQVDQlMjIlNUQlMjAlM0QlMjAocmVzdWx0JTVCJTIyZXZhbF9zY29yZV9HUFQ0JTIyJTVEJTIwLSUyMDEpJTIwJTJGJTIwNA==",highlighted:`result[<span class="hljs-string">&quot;eval_score_GPT4&quot;</span>] = result[<span class="hljs-string">&quot;eval_score_GPT4&quot;</span>].apply(<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">int</span>(x) <span class="hljs-keyword">if</span> <span class="hljs-built_in">isinstance</span>(x, <span class="hljs-built_in">str</span>) <span class="hljs-keyword">else</span> <span class="hljs-number">1</span>)
result[<span class="hljs-string">&quot;eval_score_GPT4&quot;</span>] = (result[<span class="hljs-string">&quot;eval_score_GPT4&quot;</span>] - <span class="hljs-number">1</span>) / <span class="hljs-number">4</span>`,wrap:!1}}),ss=new w({props:{code:"YXZlcmFnZV9zY29yZXMlMjAlM0QlMjByZXN1bHQuZ3JvdXBieSglMjJzZXR0aW5ncyUyMiklNUIlMjJldmFsX3Njb3JlX0dQVDQlMjIlNUQubWVhbigpJTBBYXZlcmFnZV9zY29yZXMuc29ydF92YWx1ZXMoKQ==",highlighted:`average_scores = result.groupby(<span class="hljs-string">&quot;settings&quot;</span>)[<span class="hljs-string">&quot;eval_score_GPT4&quot;</span>].mean()
average_scores.sort_values()`,wrap:!1}}),es=new c({props:{title:"Example results",local:"example-results",headingTag:"h2"}}),ns=new w({props:{code:"aW1wb3J0JTIwcGxvdGx5LmV4cHJlc3MlMjBhcyUyMHB4JTBBJTBBc2NvcmVzJTIwJTNEJTIwZGF0YXNldHMubG9hZF9kYXRhc2V0KCUyMm0tcmljJTJGcmFnX3Njb3Jlc19jb29rYm9vayUyMiUyQyUyMHNwbGl0JTNEJTIydHJhaW4lMjIpJTBBc2NvcmVzJTIwJTNEJTIwcGQuU2VyaWVzKHNjb3JlcyU1QiUyMnNjb3JlJTIyJTVEJTJDJTIwaW5kZXglM0RzY29yZXMlNUIlMjJzZXR0aW5ncyUyMiU1RCk=",highlighted:`<span class="hljs-keyword">import</span> plotly.express <span class="hljs-keyword">as</span> px

scores = datasets.load_dataset(<span class="hljs-string">&quot;m-ric/rag_scores_cookbook&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
scores = pd.Series(scores[<span class="hljs-string">&quot;score&quot;</span>], index=scores[<span class="hljs-string">&quot;settings&quot;</span>])`,wrap:!1}}),ys=new w({props:{code:"ZmlnJTIwJTNEJTIwcHguYmFyKCUwQSUyMCUyMCUyMCUyMHNjb3JlcyUyQyUwQSUyMCUyMCUyMCUyMGNvbG9yJTNEc2NvcmVzJTJDJTBBJTIwJTIwJTIwJTIwbGFiZWxzJTNEJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydmFsdWUlMjIlM0ElMjAlMjJBY2N1cmFjeSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnNldHRpbmdzJTIyJTNBJTIwJTIyQ29uZmlndXJhdGlvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMGNvbG9yX2NvbnRpbnVvdXNfc2NhbGUlM0QlMjJibHVlcmVkJTIyJTJDJTBBKSUwQWZpZy51cGRhdGVfbGF5b3V0KHclMEElMjAlMjAlMjAlMjB3aWR0aCUzRDEwMDAlMkMlMEElMjAlMjAlMjAlMjBoZWlnaHQlM0Q2MDAlMkMlMEElMjAlMjAlMjAlMjBiYXJtb2RlJTNEJTIyZ3JvdXAlMjIlMkMlMEElMjAlMjAlMjAlMjB5YXhpc19yYW5nZSUzRCU1QjAlMkMlMjAxMDAlNUQlMkMlMEElMjAlMjAlMjAlMjB0aXRsZSUzRCUyMiUzQ2IlM0VBY2N1cmFjeSUyMG9mJTIwZGlmZmVyZW50JTIwUkFHJTIwY29uZmlndXJhdGlvbnMlM0MlMkZiJTNFJTIyJTJDJTBBJTIwJTIwJTIwJTIweGF4aXNfdGl0bGUlM0QlMjJSQUclMjBzZXR0aW5ncyUyMiUyQyUwQSUyMCUyMCUyMCUyMGZvbnQlM0RkaWN0KHNpemUlM0QxNSklMkMlMEEpJTBBZmlnLmxheW91dC55YXhpcy50aWNrc3VmZml4JTIwJTNEJTIwJTIyJTI1JTIyJTBBZmlnLnVwZGF0ZV9jb2xvcmF4ZXMoc2hvd3NjYWxlJTNERmFsc2UpJTBBZmlnLnVwZGF0ZV90cmFjZXModGV4dHRlbXBsYXRlJTNEJTIyJTI1JTdCeSUzQS4xZiU3RCUyMiUyQyUyMHRleHRwb3NpdGlvbiUzRCUyMm91dHNpZGUlMjIpJTBBZmlnLnNob3coKQ==",highlighted:`fig = px.bar(
    scores,
    color=scores,
    labels={
        <span class="hljs-string">&quot;value&quot;</span>: <span class="hljs-string">&quot;Accuracy&quot;</span>,
        <span class="hljs-string">&quot;settings&quot;</span>: <span class="hljs-string">&quot;Configuration&quot;</span>,
    },
    color_continuous_scale=<span class="hljs-string">&quot;bluered&quot;</span>,
)
fig.update_layout(w
    width=<span class="hljs-number">1000</span>,
    height=<span class="hljs-number">600</span>,
    barmode=<span class="hljs-string">&quot;group&quot;</span>,
    yaxis_range=[<span class="hljs-number">0</span>, <span class="hljs-number">100</span>],
    title=<span class="hljs-string">&quot;&lt;b&gt;Accuracy of different RAG configurations&lt;/b&gt;&quot;</span>,
    xaxis_title=<span class="hljs-string">&quot;RAG settings&quot;</span>,
    font=<span class="hljs-built_in">dict</span>(size=<span class="hljs-number">15</span>),
)
fig.layout.yaxis.ticksuffix = <span class="hljs-string">&quot;%&quot;</span>
fig.update_coloraxes(showscale=<span class="hljs-literal">False</span>)
fig.update_traces(texttemplate=<span class="hljs-string">&quot;%{y:.1f}&quot;</span>, textposition=<span class="hljs-string">&quot;outside&quot;</span>)
fig.show()`,wrap:!1}}),is=new CM({props:{source:"https://github.com/huggingface/cookbook/blob/main/notebooks/en/rag_evaluation.md"}}),{c(){u=n("meta"),js=M(),ps=n("p"),ws=M(),J(d.$$.fragment),cs=M(),J(I.$$.fragment),us=M(),m=n("p"),m.innerHTML=ot,rs=M(),h=n("p"),h.textContent=pt,ds=M(),C=n("p"),C.innerHTML=Tt,Is=M(),b=n("p"),b.textContent=jt,ms=M(),Z=n("img"),hs=M(),B=n("p"),B.textContent=ct,Cs=M(),J(G.$$.fragment),bs=M(),A=n("p"),A.textContent=ut,Zs=M(),g=n("p"),g.textContent=rt,Bs=M(),f=n("ol"),f.innerHTML=dt,Gs=M(),W=n("p"),W.textContent=It,As=M(),V=n("ol"),V.innerHTML=mt,gs=M(),Q=n("p"),Q.innerHTML=ht,fs=M(),J(N.$$.fragment),Ws=M(),J(R.$$.fragment),Vs=M(),J(v.$$.fragment),Qs=M(),J(X.$$.fragment),Ns=M(),J(k.$$.fragment),Rs=M(),J(E.$$.fragment),vs=M(),J(S.$$.fragment),Xs=M(),Y=n("p"),Y.textContent=Ct,ks=M(),q=n("p"),q.textContent=bt,Es=M(),J(F.$$.fragment),Ss=M(),J(_.$$.fragment),Ys=M(),J(x.$$.fragment),qs=M(),z=n("p"),z.innerHTML=Zt,Fs=M(),J(H.$$.fragment),_s=M(),J($.$$.fragment),xs=M(),L=n("p"),L.textContent=Bt,zs=M(),D=n("p"),D.textContent=Gt,Hs=M(),J(K.$$.fragment),$s=M(),J(P.$$.fragment),Ls=M(),J(O.$$.fragment),Ds=M(),ll=n("p"),ll.textContent=At,Ks=M(),sl=n("p"),sl.innerHTML=gt,Ps=M(),el=n("ul"),el.innerHTML=ft,Os=M(),tl=n("p"),tl.innerHTML=Wt,le=M(),Ml=n("ul"),Ml.innerHTML=Vt,se=M(),al=n("p"),al.textContent=Qt,ee=M(),nl=n("p"),nl.innerHTML=Nt,te=M(),yl=n("p"),yl.textContent=Rt,Me=M(),J(Ul.$$.fragment),ae=M(),J(Jl.$$.fragment),ne=M(),il=n("p"),il.textContent=vt,ye=M(),J(ol.$$.fragment),Ue=M(),pl=n("pre"),pl.textContent=Xt,Je=M(),Tl=n("p"),Tl.textContent=kt,ie=M(),jl=n("p"),jl.textContent=Et,oe=M(),J(wl.$$.fragment),pe=M(),J(cl.$$.fragment),Te=M(),J(ul.$$.fragment),je=M(),rl=n("ul"),rl.innerHTML=St,we=M(),dl=n("p"),dl.textContent=Yt,ce=M(),Il=n("ul"),Il.innerHTML=qt,ue=M(),ml=n("p"),ml.innerHTML=Ft,re=M(),hl=n("p"),hl.innerHTML=_t,de=M(),Cl=n("blockquote"),Cl.innerHTML=xt,Ie=M(),bl=n("p"),bl.innerHTML=zt,me=M(),J(Zl.$$.fragment),he=M(),J(Bl.$$.fragment),Ce=M(),J(Gl.$$.fragment),be=M(),Al=n("p"),Al.innerHTML=Ht,Ze=M(),gl=n("blockquote"),gl.innerHTML=$t,Be=M(),fl=n("p"),fl.innerHTML=Lt,Ge=M(),Wl=n("ul"),Wl.innerHTML=Dt,Ae=M(),J(Vl.$$.fragment),ge=M(),J(Ql.$$.fragment),fe=M(),Nl=n("p"),Nl.innerHTML=Kt,We=M(),Rl=n("p"),Rl.textContent=Pt,Ve=M(),vl=n("ul"),vl.innerHTML=Ot,Qe=M(),J(Xl.$$.fragment),Ne=M(),J(kl.$$.fragment),Re=M(),J(El.$$.fragment),ve=M(),J(Sl.$$.fragment),Xe=M(),Yl=n("p"),Yl.textContent=lM,ke=M(),ql=n("p"),ql.innerHTML=sM,Ee=M(),Fl=n("p"),Fl.innerHTML=eM,Se=M(),_l=n("blockquote"),_l.innerHTML=tM,Ye=M(),xl=n("p"),xl.innerHTML=MM,qe=M(),zl=n("p"),zl.innerHTML=aM,Fe=M(),J(Hl.$$.fragment),_e=M(),J($l.$$.fragment),xe=M(),J(Ll.$$.fragment),ze=M(),Dl=n("p"),Dl.textContent=nM,He=M(),J(Kl.$$.fragment),$e=M(),J(Pl.$$.fragment),Le=M(),J(Ol.$$.fragment),De=M(),J(ls.$$.fragment),Ke=M(),J(ss.$$.fragment),Pe=M(),J(es.$$.fragment),Oe=M(),ts=n("p"),ts.innerHTML=yM,lt=M(),Ms=n("p"),Ms.textContent=UM,st=M(),as=n("p"),as.innerHTML=JM,et=M(),J(ns.$$.fragment),tt=M(),J(ys.$$.fragment),Mt=M(),r=n("img"),at=M(),Us=n("p"),Us.textContent=oM,nt=M(),Js=n("p"),Js.textContent=pM,yt=M(),J(is.$$.fragment),Ut=M(),Ts=n("p"),this.h()},l(l){const s=IM("svelte-u9bgzb",document.head);u=y(s,"META",{name:!0,content:!0}),s.forEach(e),js=a(l),ps=y(l,"P",{}),jM(ps).forEach(e),ws=a(l),i(d.$$.fragment,l),cs=a(l),i(I.$$.fragment,l),us=a(l),m=y(l,"P",{"data-svelte-h":!0}),U(m)!=="svelte-1xlqnsv"&&(m.innerHTML=ot),rs=a(l),h=y(l,"P",{"data-svelte-h":!0}),U(h)!=="svelte-2xiv25"&&(h.textContent=pt),ds=a(l),C=y(l,"P",{"data-svelte-h":!0}),U(C)!=="svelte-1460hpm"&&(C.innerHTML=Tt),Is=a(l),b=y(l,"P",{"data-svelte-h":!0}),U(b)!=="svelte-9eyhtm"&&(b.textContent=jt),ms=a(l),Z=y(l,"IMG",{src:!0,height:!0}),hs=a(l),B=y(l,"P",{"data-svelte-h":!0}),U(B)!=="svelte-1wwknqm"&&(B.textContent=ct),Cs=a(l),i(G.$$.fragment,l),bs=a(l),A=y(l,"P",{"data-svelte-h":!0}),U(A)!=="svelte-vv8zeo"&&(A.textContent=ut),Zs=a(l),g=y(l,"P",{"data-svelte-h":!0}),U(g)!=="svelte-baonz3"&&(g.textContent=rt),Bs=a(l),f=y(l,"OL",{"data-svelte-h":!0}),U(f)!=="svelte-12qpxq2"&&(f.innerHTML=dt),Gs=a(l),W=y(l,"P",{"data-svelte-h":!0}),U(W)!=="svelte-gq3s7j"&&(W.textContent=It),As=a(l),V=y(l,"OL",{"data-svelte-h":!0}),U(V)!=="svelte-qbxvp7"&&(V.innerHTML=mt),gs=a(l),Q=y(l,"P",{"data-svelte-h":!0}),U(Q)!=="svelte-15hqrk"&&(Q.innerHTML=ht),fs=a(l),i(N.$$.fragment,l),Ws=a(l),i(R.$$.fragment,l),Vs=a(l),i(v.$$.fragment,l),Qs=a(l),i(X.$$.fragment,l),Ns=a(l),i(k.$$.fragment,l),Rs=a(l),i(E.$$.fragment,l),vs=a(l),i(S.$$.fragment,l),Xs=a(l),Y=y(l,"P",{"data-svelte-h":!0}),U(Y)!=="svelte-1wbiyhw"&&(Y.textContent=Ct),ks=a(l),q=y(l,"P",{"data-svelte-h":!0}),U(q)!=="svelte-1o0ze03"&&(q.textContent=bt),Es=a(l),i(F.$$.fragment,l),Ss=a(l),i(_.$$.fragment,l),Ys=a(l),i(x.$$.fragment,l),qs=a(l),z=y(l,"P",{"data-svelte-h":!0}),U(z)!=="svelte-162gsjg"&&(z.innerHTML=Zt),Fs=a(l),i(H.$$.fragment,l),_s=a(l),i($.$$.fragment,l),xs=a(l),L=y(l,"P",{"data-svelte-h":!0}),U(L)!=="svelte-1v7z1mr"&&(L.textContent=Bt),zs=a(l),D=y(l,"P",{"data-svelte-h":!0}),U(D)!=="svelte-jsmlwo"&&(D.textContent=Gt),Hs=a(l),i(K.$$.fragment,l),$s=a(l),i(P.$$.fragment,l),Ls=a(l),i(O.$$.fragment,l),Ds=a(l),ll=y(l,"P",{"data-svelte-h":!0}),U(ll)!=="svelte-xa68th"&&(ll.textContent=At),Ks=a(l),sl=y(l,"P",{"data-svelte-h":!0}),U(sl)!=="svelte-1rn1tpr"&&(sl.innerHTML=gt),Ps=a(l),el=y(l,"UL",{"data-svelte-h":!0}),U(el)!=="svelte-mge22a"&&(el.innerHTML=ft),Os=a(l),tl=y(l,"P",{"data-svelte-h":!0}),U(tl)!=="svelte-1a9sdbs"&&(tl.innerHTML=Wt),le=a(l),Ml=y(l,"UL",{"data-svelte-h":!0}),U(Ml)!=="svelte-17ofou1"&&(Ml.innerHTML=Vt),se=a(l),al=y(l,"P",{"data-svelte-h":!0}),U(al)!=="svelte-1bu41t1"&&(al.textContent=Qt),ee=a(l),nl=y(l,"P",{"data-svelte-h":!0}),U(nl)!=="svelte-v7jbl8"&&(nl.innerHTML=Nt),te=a(l),yl=y(l,"P",{"data-svelte-h":!0}),U(yl)!=="svelte-1kwgyx1"&&(yl.textContent=Rt),Me=a(l),i(Ul.$$.fragment,l),ae=a(l),i(Jl.$$.fragment,l),ne=a(l),il=y(l,"P",{"data-svelte-h":!0}),U(il)!=="svelte-yms2wp"&&(il.textContent=vt),ye=a(l),i(ol.$$.fragment,l),Ue=a(l),pl=y(l,"PRE",{"data-svelte-h":!0}),U(pl)!=="svelte-1k8l62p"&&(pl.textContent=Xt),Je=a(l),Tl=y(l,"P",{"data-svelte-h":!0}),U(Tl)!=="svelte-1asqnia"&&(Tl.textContent=kt),ie=a(l),jl=y(l,"P",{"data-svelte-h":!0}),U(jl)!=="svelte-alkdir"&&(jl.textContent=Et),oe=a(l),i(wl.$$.fragment,l),pe=a(l),i(cl.$$.fragment,l),Te=a(l),i(ul.$$.fragment,l),je=a(l),rl=y(l,"UL",{"data-svelte-h":!0}),U(rl)!=="svelte-1fnij86"&&(rl.innerHTML=St),we=a(l),dl=y(l,"P",{"data-svelte-h":!0}),U(dl)!=="svelte-53zzzq"&&(dl.textContent=Yt),ce=a(l),Il=y(l,"UL",{"data-svelte-h":!0}),U(Il)!=="svelte-1g19xro"&&(Il.innerHTML=qt),ue=a(l),ml=y(l,"P",{"data-svelte-h":!0}),U(ml)!=="svelte-1ns6yk3"&&(ml.innerHTML=Ft),re=a(l),hl=y(l,"P",{"data-svelte-h":!0}),U(hl)!=="svelte-1hitev5"&&(hl.innerHTML=_t),de=a(l),Cl=y(l,"BLOCKQUOTE",{"data-svelte-h":!0}),U(Cl)!=="svelte-w4wn00"&&(Cl.innerHTML=xt),Ie=a(l),bl=y(l,"P",{"data-svelte-h":!0}),U(bl)!=="svelte-1toh8cv"&&(bl.innerHTML=zt),me=a(l),i(Zl.$$.fragment,l),he=a(l),i(Bl.$$.fragment,l),Ce=a(l),i(Gl.$$.fragment,l),be=a(l),Al=y(l,"P",{"data-svelte-h":!0}),U(Al)!=="svelte-xymbek"&&(Al.innerHTML=Ht),Ze=a(l),gl=y(l,"BLOCKQUOTE",{"data-svelte-h":!0}),U(gl)!=="svelte-1k8ukvt"&&(gl.innerHTML=$t),Be=a(l),fl=y(l,"P",{"data-svelte-h":!0}),U(fl)!=="svelte-1hqptzf"&&(fl.innerHTML=Lt),Ge=a(l),Wl=y(l,"UL",{"data-svelte-h":!0}),U(Wl)!=="svelte-1w0jpt6"&&(Wl.innerHTML=Dt),Ae=a(l),i(Vl.$$.fragment,l),ge=a(l),i(Ql.$$.fragment,l),fe=a(l),Nl=y(l,"P",{"data-svelte-h":!0}),U(Nl)!=="svelte-1m0x0qs"&&(Nl.innerHTML=Kt),We=a(l),Rl=y(l,"P",{"data-svelte-h":!0}),U(Rl)!=="svelte-1e3sl85"&&(Rl.textContent=Pt),Ve=a(l),vl=y(l,"UL",{"data-svelte-h":!0}),U(vl)!=="svelte-1scj3jf"&&(vl.innerHTML=Ot),Qe=a(l),i(Xl.$$.fragment,l),Ne=a(l),i(kl.$$.fragment,l),Re=a(l),i(El.$$.fragment,l),ve=a(l),i(Sl.$$.fragment,l),Xe=a(l),Yl=y(l,"P",{"data-svelte-h":!0}),U(Yl)!=="svelte-z2wcut"&&(Yl.textContent=lM),ke=a(l),ql=y(l,"P",{"data-svelte-h":!0}),U(ql)!=="svelte-1tiikby"&&(ql.innerHTML=sM),Ee=a(l),Fl=y(l,"P",{"data-svelte-h":!0}),U(Fl)!=="svelte-13s04hx"&&(Fl.innerHTML=eM),Se=a(l),_l=y(l,"BLOCKQUOTE",{"data-svelte-h":!0}),U(_l)!=="svelte-jkfwyy"&&(_l.innerHTML=tM),Ye=a(l),xl=y(l,"P",{"data-svelte-h":!0}),U(xl)!=="svelte-qutlt3"&&(xl.innerHTML=MM),qe=a(l),zl=y(l,"P",{"data-svelte-h":!0}),U(zl)!=="svelte-1sle7mx"&&(zl.innerHTML=aM),Fe=a(l),i(Hl.$$.fragment,l),_e=a(l),i($l.$$.fragment,l),xe=a(l),i(Ll.$$.fragment,l),ze=a(l),Dl=y(l,"P",{"data-svelte-h":!0}),U(Dl)!=="svelte-ieknxj"&&(Dl.textContent=nM),He=a(l),i(Kl.$$.fragment,l),$e=a(l),i(Pl.$$.fragment,l),Le=a(l),i(Ol.$$.fragment,l),De=a(l),i(ls.$$.fragment,l),Ke=a(l),i(ss.$$.fragment,l),Pe=a(l),i(es.$$.fragment,l),Oe=a(l),ts=y(l,"P",{"data-svelte-h":!0}),U(ts)!=="svelte-5h9xp6"&&(ts.innerHTML=yM),lt=a(l),Ms=y(l,"P",{"data-svelte-h":!0}),U(Ms)!=="svelte-6vrle6"&&(Ms.textContent=UM),st=a(l),as=y(l,"P",{"data-svelte-h":!0}),U(as)!=="svelte-1l99caq"&&(as.innerHTML=JM),et=a(l),i(ns.$$.fragment,l),tt=a(l),i(ys.$$.fragment,l),Mt=a(l),r=y(l,"IMG",{src:!0,height:!0,width:!0}),at=a(l),Us=y(l,"P",{"data-svelte-h":!0}),U(Us)!=="svelte-i71q2k"&&(Us.textContent=oM),nt=a(l),Js=y(l,"P",{"data-svelte-h":!0}),U(Js)!=="svelte-5zkqt2"&&(Js.textContent=pM),yt=a(l),i(is.$$.fragment,l),Ut=a(l),Ts=y(l,"P",{}),jM(Ts).forEach(e),this.h()},h(){os(u,"name","hf:doc:metadata"),os(u,"content",ZM),TM(Z.src,wt="https://huggingface.co/datasets/huggingface/cookbook-images/resolve/main/RAG_workflow.png")||os(Z,"src",wt),os(Z,"height","700"),TM(r.src,iM="https://huggingface.co/datasets/huggingface/cookbook-images/resolve/main/RAG_settings_accuracy.png")||os(r,"src",iM),os(r,"height","500"),os(r,"width","800")},m(l,s){mM(document.head,u),t(l,js,s),t(l,ps,s),t(l,ws,s),o(d,l,s),t(l,cs,s),o(I,l,s),t(l,us,s),t(l,m,s),t(l,rs,s),t(l,h,s),t(l,ds,s),t(l,C,s),t(l,Is,s),t(l,b,s),t(l,ms,s),t(l,Z,s),t(l,hs,s),t(l,B,s),t(l,Cs,s),o(G,l,s),t(l,bs,s),t(l,A,s),t(l,Zs,s),t(l,g,s),t(l,Bs,s),t(l,f,s),t(l,Gs,s),t(l,W,s),t(l,As,s),t(l,V,s),t(l,gs,s),t(l,Q,s),t(l,fs,s),o(N,l,s),t(l,Ws,s),o(R,l,s),t(l,Vs,s),o(v,l,s),t(l,Qs,s),o(X,l,s),t(l,Ns,s),o(k,l,s),t(l,Rs,s),o(E,l,s),t(l,vs,s),o(S,l,s),t(l,Xs,s),t(l,Y,s),t(l,ks,s),t(l,q,s),t(l,Es,s),o(F,l,s),t(l,Ss,s),o(_,l,s),t(l,Ys,s),o(x,l,s),t(l,qs,s),t(l,z,s),t(l,Fs,s),o(H,l,s),t(l,_s,s),o($,l,s),t(l,xs,s),t(l,L,s),t(l,zs,s),t(l,D,s),t(l,Hs,s),o(K,l,s),t(l,$s,s),o(P,l,s),t(l,Ls,s),o(O,l,s),t(l,Ds,s),t(l,ll,s),t(l,Ks,s),t(l,sl,s),t(l,Ps,s),t(l,el,s),t(l,Os,s),t(l,tl,s),t(l,le,s),t(l,Ml,s),t(l,se,s),t(l,al,s),t(l,ee,s),t(l,nl,s),t(l,te,s),t(l,yl,s),t(l,Me,s),o(Ul,l,s),t(l,ae,s),o(Jl,l,s),t(l,ne,s),t(l,il,s),t(l,ye,s),o(ol,l,s),t(l,Ue,s),t(l,pl,s),t(l,Je,s),t(l,Tl,s),t(l,ie,s),t(l,jl,s),t(l,oe,s),o(wl,l,s),t(l,pe,s),o(cl,l,s),t(l,Te,s),o(ul,l,s),t(l,je,s),t(l,rl,s),t(l,we,s),t(l,dl,s),t(l,ce,s),t(l,Il,s),t(l,ue,s),t(l,ml,s),t(l,re,s),t(l,hl,s),t(l,de,s),t(l,Cl,s),t(l,Ie,s),t(l,bl,s),t(l,me,s),o(Zl,l,s),t(l,he,s),o(Bl,l,s),t(l,Ce,s),o(Gl,l,s),t(l,be,s),t(l,Al,s),t(l,Ze,s),t(l,gl,s),t(l,Be,s),t(l,fl,s),t(l,Ge,s),t(l,Wl,s),t(l,Ae,s),o(Vl,l,s),t(l,ge,s),o(Ql,l,s),t(l,fe,s),t(l,Nl,s),t(l,We,s),t(l,Rl,s),t(l,Ve,s),t(l,vl,s),t(l,Qe,s),o(Xl,l,s),t(l,Ne,s),o(kl,l,s),t(l,Re,s),o(El,l,s),t(l,ve,s),o(Sl,l,s),t(l,Xe,s),t(l,Yl,s),t(l,ke,s),t(l,ql,s),t(l,Ee,s),t(l,Fl,s),t(l,Se,s),t(l,_l,s),t(l,Ye,s),t(l,xl,s),t(l,qe,s),t(l,zl,s),t(l,Fe,s),o(Hl,l,s),t(l,_e,s),o($l,l,s),t(l,xe,s),o(Ll,l,s),t(l,ze,s),t(l,Dl,s),t(l,He,s),o(Kl,l,s),t(l,$e,s),o(Pl,l,s),t(l,Le,s),o(Ol,l,s),t(l,De,s),o(ls,l,s),t(l,Ke,s),o(ss,l,s),t(l,Pe,s),o(es,l,s),t(l,Oe,s),t(l,ts,s),t(l,lt,s),t(l,Ms,s),t(l,st,s),t(l,as,s),t(l,et,s),o(ns,l,s),t(l,tt,s),o(ys,l,s),t(l,Mt,s),t(l,r,s),t(l,at,s),t(l,Us,s),t(l,nt,s),t(l,Js,s),t(l,yt,s),o(is,l,s),t(l,Ut,s),t(l,Ts,s),Jt=!0},p:cM,i(l){Jt||(p(d.$$.fragment,l),p(I.$$.fragment,l),p(G.$$.fragment,l),p(N.$$.fragment,l),p(R.$$.fragment,l),p(v.$$.fragment,l),p(X.$$.fragment,l),p(k.$$.fragment,l),p(E.$$.fragment,l),p(S.$$.fragment,l),p(F.$$.fragment,l),p(_.$$.fragment,l),p(x.$$.fragment,l),p(H.$$.fragment,l),p($.$$.fragment,l),p(K.$$.fragment,l),p(P.$$.fragment,l),p(O.$$.fragment,l),p(Ul.$$.fragment,l),p(Jl.$$.fragment,l),p(ol.$$.fragment,l),p(wl.$$.fragment,l),p(cl.$$.fragment,l),p(ul.$$.fragment,l),p(Zl.$$.fragment,l),p(Bl.$$.fragment,l),p(Gl.$$.fragment,l),p(Vl.$$.fragment,l),p(Ql.$$.fragment,l),p(Xl.$$.fragment,l),p(kl.$$.fragment,l),p(El.$$.fragment,l),p(Sl.$$.fragment,l),p(Hl.$$.fragment,l),p($l.$$.fragment,l),p(Ll.$$.fragment,l),p(Kl.$$.fragment,l),p(Pl.$$.fragment,l),p(Ol.$$.fragment,l),p(ls.$$.fragment,l),p(ss.$$.fragment,l),p(es.$$.fragment,l),p(ns.$$.fragment,l),p(ys.$$.fragment,l),p(is.$$.fragment,l),Jt=!0)},o(l){T(d.$$.fragment,l),T(I.$$.fragment,l),T(G.$$.fragment,l),T(N.$$.fragment,l),T(R.$$.fragment,l),T(v.$$.fragment,l),T(X.$$.fragment,l),T(k.$$.fragment,l),T(E.$$.fragment,l),T(S.$$.fragment,l),T(F.$$.fragment,l),T(_.$$.fragment,l),T(x.$$.fragment,l),T(H.$$.fragment,l),T($.$$.fragment,l),T(K.$$.fragment,l),T(P.$$.fragment,l),T(O.$$.fragment,l),T(Ul.$$.fragment,l),T(Jl.$$.fragment,l),T(ol.$$.fragment,l),T(wl.$$.fragment,l),T(cl.$$.fragment,l),T(ul.$$.fragment,l),T(Zl.$$.fragment,l),T(Bl.$$.fragment,l),T(Gl.$$.fragment,l),T(Vl.$$.fragment,l),T(Ql.$$.fragment,l),T(Xl.$$.fragment,l),T(kl.$$.fragment,l),T(El.$$.fragment,l),T(Sl.$$.fragment,l),T(Hl.$$.fragment,l),T($l.$$.fragment,l),T(Ll.$$.fragment,l),T(Kl.$$.fragment,l),T(Pl.$$.fragment,l),T(Ol.$$.fragment,l),T(ls.$$.fragment,l),T(ss.$$.fragment,l),T(es.$$.fragment,l),T(ns.$$.fragment,l),T(ys.$$.fragment,l),T(is.$$.fragment,l),Jt=!1},d(l){l&&(e(js),e(ps),e(ws),e(cs),e(us),e(m),e(rs),e(h),e(ds),e(C),e(Is),e(b),e(ms),e(Z),e(hs),e(B),e(Cs),e(bs),e(A),e(Zs),e(g),e(Bs),e(f),e(Gs),e(W),e(As),e(V),e(gs),e(Q),e(fs),e(Ws),e(Vs),e(Qs),e(Ns),e(Rs),e(vs),e(Xs),e(Y),e(ks),e(q),e(Es),e(Ss),e(Ys),e(qs),e(z),e(Fs),e(_s),e(xs),e(L),e(zs),e(D),e(Hs),e($s),e(Ls),e(Ds),e(ll),e(Ks),e(sl),e(Ps),e(el),e(Os),e(tl),e(le),e(Ml),e(se),e(al),e(ee),e(nl),e(te),e(yl),e(Me),e(ae),e(ne),e(il),e(ye),e(Ue),e(pl),e(Je),e(Tl),e(ie),e(jl),e(oe),e(pe),e(Te),e(je),e(rl),e(we),e(dl),e(ce),e(Il),e(ue),e(ml),e(re),e(hl),e(de),e(Cl),e(Ie),e(bl),e(me),e(he),e(Ce),e(be),e(Al),e(Ze),e(gl),e(Be),e(fl),e(Ge),e(Wl),e(Ae),e(ge),e(fe),e(Nl),e(We),e(Rl),e(Ve),e(vl),e(Qe),e(Ne),e(Re),e(ve),e(Xe),e(Yl),e(ke),e(ql),e(Ee),e(Fl),e(Se),e(_l),e(Ye),e(xl),e(qe),e(zl),e(Fe),e(_e),e(xe),e(ze),e(Dl),e(He),e($e),e(Le),e(De),e(Ke),e(Pe),e(Oe),e(ts),e(lt),e(Ms),e(st),e(as),e(et),e(tt),e(Mt),e(r),e(at),e(Us),e(nt),e(Js),e(yt),e(Ut),e(Ts)),e(u),j(d,l),j(I,l),j(G,l),j(N,l),j(R,l),j(v,l),j(X,l),j(k,l),j(E,l),j(S,l),j(F,l),j(_,l),j(x,l),j(H,l),j($,l),j(K,l),j(P,l),j(O,l),j(Ul,l),j(Jl,l),j(ol,l),j(wl,l),j(cl,l),j(ul,l),j(Zl,l),j(Bl,l),j(Gl,l),j(Vl,l),j(Ql,l),j(Xl,l),j(kl,l),j(El,l),j(Sl,l),j(Hl,l),j($l,l),j(Ll,l),j(Kl,l),j(Pl,l),j(Ol,l),j(ls,l),j(ss,l),j(es,l),j(ns,l),j(ys,l),j(is,l)}}}const ZM='{"title":"RAG Evaluation","local":"rag-evaluation","sections":[{"title":"Evaluating RAG performance","local":"evaluating-rag-performance","sections":[],"depth":3},{"title":"Load your knowledge base","local":"load-your-knowledge-base","sections":[],"depth":3}],"depth":1}';function BM(it){return uM(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class WM extends rM{constructor(u){super(),dM(this,u,BM,bM,wM,{})}}export{WM as component};
