import{s as Ee,a as _e,n as Se,o as Ie}from"../chunks/scheduler.3a17fb72.js";import{S as Ze,i as Pe,e as f,s as i,c as s,h as He,a as g,d as a,b as n,f as he,g as r,j as B,k as h,l as Fe,m as l,n as o,t as m,o as u,p}from"../chunks/index.093f8863.js";import{C as Ge,H as d,E as ze}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.2631eeef.js";import{C as Ue}from"../chunks/CodeBlock.b3d2386f.js";function Je(ve){let v,j,W,A,b,Q,w,V,y,be=`🤗 Lighteval is your all-in-one toolkit for evaluating Large Language Models
(LLMs) across multiple backends with ease. Dive deep into your model’s
performance by saving and exploring detailed, sample-by-sample results to debug
and see how your models stack up.`,D,k,X,C,Y,x,we="Evaluate your models using the most popular and efficient inference backends:",K,M,ye=`<li><code>eval</code>: Use <a href="https://inspect.aisi.org.uk/" rel="nofollow">inspect-ai</a> as backend to evaluate and inspect your models ! (prefered way)</li> <li><code>transformers</code>: Evaluate models on CPU or one or more GPUs using <a href="https://github.com/huggingface/transformers" rel="nofollow">🤗
Accelerate</a></li> <li><code>nanotron</code>: Evaluate models in distributed settings using <a href="https://github.com/huggingface/nanotron" rel="nofollow">⚡️
Nanotron</a></li> <li><code>vllm</code>: Evaluate models on one or more GPUs using <a href="https://github.com/vllm-project/vllm" rel="nofollow">🚀
VLLM</a></li> <li><code>custom</code>: Evaluate custom models (can be anything)</li> <li><code>sglang</code>: Evaluate models using <a href="https://github.com/sgl-project/sglang" rel="nofollow">SGLang</a> as backend</li> <li><code>inference-endpoint</code>: Evaluate models using Hugging Face’s <a href="https://huggingface.co/inference-endpoints/dedicated" rel="nofollow">Inference Endpoints API</a></li> <li><code>tgi</code>: Evaluate models using <a href="https://huggingface.co/docs/text-generation-inference/en/index" rel="nofollow">🔗 Text Generation Inference</a> running locally</li> <li><code>litellm</code>: Evaluate models on any compatible API using <a href="https://www.litellm.ai/" rel="nofollow">LiteLLM</a></li> <li><code>inference-providers</code>: Evaluate models using <a href="https://huggingface.co/docs/inference-providers/en/index" rel="nofollow">HuggingFace’s inference providers</a> as backend**: Distributed training and evaluation</li>`,N,T,O,L,ke="<li><strong>Extensive Task Library</strong>: 1000s pre-built evaluation tasks</li> <li><strong>Custom Task Creation</strong>: Build your own evaluation tasks</li> <li><strong>Flexible Metrics</strong>: Support for custom metrics and scoring</li> <li><strong>Detailed Analysis</strong>: Sample-by-sample results for deep insights</li>",ee,_,te,U,Ce=`Customization at your fingertips: create <a href="adding-a-custom-task">new tasks</a>,
<a href="adding-a-new-metric">metrics</a> or <a href="evaluating-a-custom-model">model</a> tailored to your needs, or browse all our existing tasks and metrics.`,ae,E,le,S,xe="Seamlessly experiment, benchmark, and store your results on the Hugging Face Hub, S3, or locally.",ie,I,ne,Z,se,P,re,H,oe,F,me,$,Me,ue,G,pe,z,fe,J,Te="Resulting Space:",ge,c,Le,de,q,$e,R,ce;return b=new Ge({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),w=new d({props:{title:"Lighteval",local:"lighteval",headingTag:"h1"}}),k=new d({props:{title:"Key Features",local:"key-features",headingTag:"h2"}}),C=new d({props:{title:"🚀 Multi-Backend Support",local:"-multi-backend-support",headingTag:"h3"}}),T=new d({props:{title:"📊 Comprehensive Evaluation",local:"-comprehensive-evaluation",headingTag:"h3"}}),_=new d({props:{title:"🔧 Easy Customization",local:"-easy-customization",headingTag:"h3"}}),E=new d({props:{title:"☁️ Seamless Integration",local:"-seamless-integration",headingTag:"h3"}}),I=new d({props:{title:"Quick Start",local:"quick-start",headingTag:"h2"}}),Z=new d({props:{title:"Installation",local:"installation",headingTag:"h3"}}),P=new Ue({props:{code:"cGlwJTIwaW5zdGFsbCUyMGxpZ2h0ZXZhbA==",highlighted:"pip install lighteval",wrap:!1}}),H=new d({props:{title:"Basic Usage",local:"basic-usage",headingTag:"h3"}}),F=new d({props:{title:"Find a task",local:"find-a-task",headingTag:"h4"}}),G=new d({props:{title:"Run your benchmark and push details to the hub",local:"run-your-benchmark-and-push-details-to-the-hub",headingTag:"h4"}}),z=new Ue({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMCUyMmhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUyMiUyMCU1QyUwQSUyMCUyMCUyMmxpZ2h0ZXZhbCU3Q2dwcWElM0FkaWFtb25kJTdDMCUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tYnVuZGxlLWRpciUyMGdwdC1vc3MtYnVuZGxlJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1yZXBvLWlkJTIwT3BlbkV2YWxzJTJGZXZhbHM=",highlighted:`lighteval <span class="hljs-built_in">eval</span> <span class="hljs-string">&quot;hf-inference-providers/openai/gpt-oss-20b&quot;</span> \\
  <span class="hljs-string">&quot;lighteval|gpqa:diamond|0&quot;</span> \\
    --bundle-dir gpt-oss-bundle \\
    --repo-id OpenEvals/evals`,wrap:!1}}),q=new ze({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/index.mdx"}}),{c(){v=f("meta"),j=i(),W=f("p"),A=i(),s(b.$$.fragment),Q=i(),s(w.$$.fragment),V=i(),y=f("p"),y.textContent=be,D=i(),s(k.$$.fragment),X=i(),s(C.$$.fragment),Y=i(),x=f("p"),x.textContent=we,K=i(),M=f("ul"),M.innerHTML=ye,N=i(),s(T.$$.fragment),O=i(),L=f("ul"),L.innerHTML=ke,ee=i(),s(_.$$.fragment),te=i(),U=f("p"),U.innerHTML=Ce,ae=i(),s(E.$$.fragment),le=i(),S=f("p"),S.textContent=xe,ie=i(),s(I.$$.fragment),ne=i(),s(Z.$$.fragment),se=i(),s(P.$$.fragment),re=i(),s(H.$$.fragment),oe=i(),s(F.$$.fragment),me=i(),$=f("iframe"),ue=i(),s(G.$$.fragment),pe=i(),s(z.$$.fragment),fe=i(),J=f("p"),J.textContent=Te,ge=i(),c=f("iframe"),de=i(),s(q.$$.fragment),$e=i(),R=f("p"),this.h()},l(e){const t=He("svelte-u9bgzb",document.head);v=g(t,"META",{name:!0,content:!0}),t.forEach(a),j=n(e),W=g(e,"P",{}),he(W).forEach(a),A=n(e),r(b.$$.fragment,e),Q=n(e),r(w.$$.fragment,e),V=n(e),y=g(e,"P",{"data-svelte-h":!0}),B(y)!=="svelte-5jd4gg"&&(y.textContent=be),D=n(e),r(k.$$.fragment,e),X=n(e),r(C.$$.fragment,e),Y=n(e),x=g(e,"P",{"data-svelte-h":!0}),B(x)!=="svelte-hqmq3z"&&(x.textContent=we),K=n(e),M=g(e,"UL",{"data-svelte-h":!0}),B(M)!=="svelte-d6biun"&&(M.innerHTML=ye),N=n(e),r(T.$$.fragment,e),O=n(e),L=g(e,"UL",{"data-svelte-h":!0}),B(L)!=="svelte-1kvd8am"&&(L.innerHTML=ke),ee=n(e),r(_.$$.fragment,e),te=n(e),U=g(e,"P",{"data-svelte-h":!0}),B(U)!=="svelte-d94dpm"&&(U.innerHTML=Ce),ae=n(e),r(E.$$.fragment,e),le=n(e),S=g(e,"P",{"data-svelte-h":!0}),B(S)!=="svelte-12kzs6l"&&(S.textContent=xe),ie=n(e),r(I.$$.fragment,e),ne=n(e),r(Z.$$.fragment,e),se=n(e),r(P.$$.fragment,e),re=n(e),r(H.$$.fragment,e),oe=n(e),r(F.$$.fragment,e),me=n(e),$=g(e,"IFRAME",{src:!0,frameborder:!0,width:!0,height:!0}),he($).forEach(a),ue=n(e),r(G.$$.fragment,e),pe=n(e),r(z.$$.fragment,e),fe=n(e),J=g(e,"P",{"data-svelte-h":!0}),B(J)!=="svelte-xmt8gz"&&(J.textContent=Te),ge=n(e),c=g(e,"IFRAME",{src:!0,frameborder:!0,width:!0,height:!0}),he(c).forEach(a),de=n(e),r(q.$$.fragment,e),$e=n(e),R=g(e,"P",{}),he(R).forEach(a),this.h()},h(){h(v,"name","hf:doc:metadata"),h(v,"content",qe),_e($.src,Me="https://openevals-open-benchmark-index.hf.space")||h($,"src",Me),h($,"frameborder","0"),h($,"width","850"),h($,"height","450"),_e(c.src,Le="https://openevals-evals.static.hf.space")||h(c,"src",Le),h(c,"frameborder","0"),h(c,"width","850"),h(c,"height","450")},m(e,t){Fe(document.head,v),l(e,j,t),l(e,W,t),l(e,A,t),o(b,e,t),l(e,Q,t),o(w,e,t),l(e,V,t),l(e,y,t),l(e,D,t),o(k,e,t),l(e,X,t),o(C,e,t),l(e,Y,t),l(e,x,t),l(e,K,t),l(e,M,t),l(e,N,t),o(T,e,t),l(e,O,t),l(e,L,t),l(e,ee,t),o(_,e,t),l(e,te,t),l(e,U,t),l(e,ae,t),o(E,e,t),l(e,le,t),l(e,S,t),l(e,ie,t),o(I,e,t),l(e,ne,t),o(Z,e,t),l(e,se,t),o(P,e,t),l(e,re,t),o(H,e,t),l(e,oe,t),o(F,e,t),l(e,me,t),l(e,$,t),l(e,ue,t),o(G,e,t),l(e,pe,t),o(z,e,t),l(e,fe,t),l(e,J,t),l(e,ge,t),l(e,c,t),l(e,de,t),o(q,e,t),l(e,$e,t),l(e,R,t),ce=!0},p:Se,i(e){ce||(m(b.$$.fragment,e),m(w.$$.fragment,e),m(k.$$.fragment,e),m(C.$$.fragment,e),m(T.$$.fragment,e),m(_.$$.fragment,e),m(E.$$.fragment,e),m(I.$$.fragment,e),m(Z.$$.fragment,e),m(P.$$.fragment,e),m(H.$$.fragment,e),m(F.$$.fragment,e),m(G.$$.fragment,e),m(z.$$.fragment,e),m(q.$$.fragment,e),ce=!0)},o(e){u(b.$$.fragment,e),u(w.$$.fragment,e),u(k.$$.fragment,e),u(C.$$.fragment,e),u(T.$$.fragment,e),u(_.$$.fragment,e),u(E.$$.fragment,e),u(I.$$.fragment,e),u(Z.$$.fragment,e),u(P.$$.fragment,e),u(H.$$.fragment,e),u(F.$$.fragment,e),u(G.$$.fragment,e),u(z.$$.fragment,e),u(q.$$.fragment,e),ce=!1},d(e){e&&(a(j),a(W),a(A),a(Q),a(V),a(y),a(D),a(X),a(Y),a(x),a(K),a(M),a(N),a(O),a(L),a(ee),a(te),a(U),a(ae),a(le),a(S),a(ie),a(ne),a(se),a(re),a(oe),a(me),a($),a(ue),a(pe),a(fe),a(J),a(ge),a(c),a(de),a($e),a(R)),a(v),p(b,e),p(w,e),p(k,e),p(C,e),p(T,e),p(_,e),p(E,e),p(I,e),p(Z,e),p(P,e),p(H,e),p(F,e),p(G,e),p(z,e),p(q,e)}}}const qe='{"title":"Lighteval","local":"lighteval","sections":[{"title":"Key Features","local":"key-features","sections":[{"title":"🚀 Multi-Backend Support","local":"-multi-backend-support","sections":[],"depth":3},{"title":"📊 Comprehensive Evaluation","local":"-comprehensive-evaluation","sections":[],"depth":3},{"title":"🔧 Easy Customization","local":"-easy-customization","sections":[],"depth":3},{"title":"☁️ Seamless Integration","local":"-seamless-integration","sections":[],"depth":3}],"depth":2},{"title":"Quick Start","local":"quick-start","sections":[{"title":"Installation","local":"installation","sections":[],"depth":3},{"title":"Basic Usage","local":"basic-usage","sections":[{"title":"Find a task","local":"find-a-task","sections":[],"depth":4},{"title":"Run your benchmark and push details to the hub","local":"run-your-benchmark-and-push-details-to-the-hub","sections":[],"depth":4}],"depth":3}],"depth":2}],"depth":1}';function Be(ve){return Ie(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Qe extends Ze{constructor(v){super(),Pe(this,v,Be,Je,Ee,{})}}export{Qe as component};
