import{s as nt,a as st,n as it,o as mt}from"../chunks/scheduler.3a17fb72.js";import{S as pt,i as rt,e as i,s as a,c as r,h as Mt,a as m,d as l,b as n,f as _e,g as M,j as u,k as p,l as ut,m as s,n as f,t as y,o as c,p as o}from"../chunks/index.093f8863.js";import{C as ft,H as at,E as yt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.8d2491fb.js";import{C as b}from"../chunks/CodeBlock.ac3e5870.js";function ct(Ve){let T,ee,D,te,W,le,I,se,k,Ae=`Pick the right benchmarks with our benchmark finder:
Search by language, task type, dataset name, or keywords.`,ae,$,Re="<p>Not all tasks are compatible with inspect-ai’s API as of yet, we are working on converting all of them !</p>",ne,h,Be,ie,G,Ye="Once you’ve chosen a benchmark, run it with <code>lighteval eval</code>. Below are examples for common setups.",me,x,pe,S,ze="<li>Evaluate a model via Hugging Face Inference Providers.</li>",re,H,Me,v,Ee="<li>Run multiple evals at the same time.</li>",ue,_,fe,d,Ne="<li>Compare providers for the same model.</li>",ye,V,ce,U,Qe="<li>Evaluate a vLLM or SGLang model.</li>",oe,A,be,g,Fe="<li>See the impact of few-shot on your model.</li>",he,R,we,Z,Xe="<li>Optimize custom server connections.</li>",Te,B,$e,C,qe="<li>Use multiple epochs for more reliable results.</li>",ve,Y,de,j,Oe="<li>Push to the Hub to share results.</li>",Ue,z,ge,E,Pe="Resulting Space:",Ze,w,De,Ce,L,Ke="<li>Change model behaviour</li>",je,N,et="You can use any argument defined in inspect-ai’s API.",Le,Q,Je,J,tt="<li>Use model-args to use any inference provider specific argument.</li>",We,F,Ie,X,ke,q,lt="LightEval prints a per-model results table:",Ge,O,xe,P,Se,K,He;return W=new ft({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),I=new at({props:{title:"Evaluate your model with Inspect-AI",local:"evaluate-your-model-with-inspect-ai",headingTag:"h1"}}),x=new at({props:{title:"Examples",local:"examples",headingTag:"h3"}}),H=new b({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMCUyMmhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUyMiUyMGdwcWElM0FkaWFtb25k",highlighted:'lighteval <span class="hljs-built_in">eval</span> <span class="hljs-string">&quot;hf-inference-providers/openai/gpt-oss-20b&quot;</span> gpqa:diamond',wrap:!1}}),_=new b({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMCUyMmhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUyMiUyMGdwcWElM0FkaWFtb25kJTJDYWltZTI1",highlighted:'lighteval <span class="hljs-built_in">eval</span> <span class="hljs-string">&quot;hf-inference-providers/openai/gpt-oss-20b&quot;</span> gpqa:diamond,aime25',wrap:!1}}),V=new b({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMCU1QyUwQSUyMCUyMCUyMCUyMGhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUzQWZpcmV3b3Jrcy1haSUyMCU1QyUwQSUyMCUyMCUyMCUyMGhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUzQXRvZ2V0aGVyJTIwJTVDJTBBJTIwJTIwJTIwJTIwaGYtaW5mZXJlbmNlLXByb3ZpZGVycyUyRm9wZW5haSUyRmdwdC1vc3MtMjBiJTNBbmViaXVzJTIwJTVDJTBBJTIwJTIwJTIwJTIwZ3BxYSUzQWRpYW1vbmQ=",highlighted:`lighteval <span class="hljs-built_in">eval</span> \\
    hf-inference-providers/openai/gpt-oss-20b:fireworks-ai \\
    hf-inference-providers/openai/gpt-oss-20b:together \\
    hf-inference-providers/openai/gpt-oss-20b:nebius \\
    gpqa:diamond`,wrap:!1}}),A=new b({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMHZsbG0lMkZIdWdnaW5nRmFjZVRCJTJGU21vbExNLTEzNU0tSW5zdHJ1Y3QlMjBncHFhJTNBZGlhbW9uZA==",highlighted:'lighteval <span class="hljs-built_in">eval</span> vllm/HuggingFaceTB/SmolLM-135M-Instruct gpqa:diamond',wrap:!1}}),R=new b({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMGhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUyMCUyMmdzbThrJTdDMCUyQ2dzbThrJTdDNSUyMg==",highlighted:'lighteval <span class="hljs-built_in">eval</span> hf-inference-providers/openai/gpt-oss-20b <span class="hljs-string">&quot;gsm8k|0,gsm8k|5&quot;</span>',wrap:!1}}),B=new b({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMGhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUyMGdzbThrJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tYXgtY29ubmVjdGlvbnMlMjA1MCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tdGltZW91dCUyMDMwJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1yZXRyeS1vbi1lcnJvciUyMDElMjAlNUMlMEElMjAlMjAlMjAlMjAtLW1heC1yZXRyaWVzJTIwMSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbWF4LXNhbXBsZXMlMjAxMA==",highlighted:`lighteval <span class="hljs-built_in">eval</span> hf-inference-providers/openai/gpt-oss-20b gsm8k \\
    --max-connections 50 \\
    --<span class="hljs-built_in">timeout</span> 30 \\
    --retry-on-error 1 \\
    --max-retries 1 \\
    --max-samples 10`,wrap:!1}}),Y=new b({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMGhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUyMGFpbWUyNSUyMC0tZXBvY2hzJTIwMTYlMjAtLWVwb2Nocy1yZWR1Y2VyJTIwJTIycGFzc19hdF80JTIy",highlighted:'lighteval <span class="hljs-built_in">eval</span> hf-inference-providers/openai/gpt-oss-20b aime25 --epochs 16 --epochs-reducer <span class="hljs-string">&quot;pass_at_4&quot;</span>',wrap:!1}}),z=new b({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMGhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUyMGhsZSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tYnVuZGxlLWRpciUyMGdwdC1vc3MtYnVuZGxlJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1yZXBvLWlkJTIwT3BlbkV2YWxzJTJGZXZhbHMlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW1heC1zYW1wbGVzJTIwMTAw",highlighted:`lighteval <span class="hljs-built_in">eval</span> hf-inference-providers/openai/gpt-oss-20b hle \\
    --bundle-dir gpt-oss-bundle \\
    --repo-id OpenEvals/evals \\
    --max-samples 100`,wrap:!1}}),Q=new b({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMGhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUyMGFpbWUyNSUyMC0tdGVtcGVyYXR1cmUlMjAwLjE=",highlighted:'lighteval <span class="hljs-built_in">eval</span> hf-inference-providers/openai/gpt-oss-20b aime25 --temperature 0.1',wrap:!1}}),F=new b({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMGdvb2dsZSUyRmdlbWluaS0yLjUtcHJvJTIwYWltZTI1JTIwLS1tb2RlbC1hcmdzJTIwbG9jYXRpb24lM0R1cy1lYXN0NQ==",highlighted:'lighteval <span class="hljs-built_in">eval</span> google/gemini-2.5-pro aime25 --model-args location=us-east5',wrap:!1}}),X=new b({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMG9wZW5haSUyRmdwdC00byUyMGdwcWElM0FkaWFtb25kJTIwLS1tb2RlbC1hcmdzJTIwc2VydmljZV90aWVyJTNEZmxleCUyQ2NsaWVudF90aW1lb3V0JTNEMTIwMA==",highlighted:'lighteval <span class="hljs-built_in">eval</span> openai/gpt-4o gpqa:diamond --model-args service_tier=flex,client_timeout=1200',wrap:!1}}),O=new b({props:{code:"Q29tcGxldGVkJTIwYWxsJTIwdGFza3MlMjBpbiUyMCdsaWdodGV2YWwtbG9ncyclMjBzdWNjZXNzZnVsbHklMEElMEElN0MlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBNb2RlbCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3Q2dwcWElN0NncHFhJTNBZGlhbW9uZCU3QyUwQSU3Qy0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLSU3Qy0tLSUzQSU3Qy0tLS0tLS0tLS0tJTNBJTdDJTBBJTdDdmxsbSUyRkh1Z2dpbmdGYWNlVEIlMkZTbW9sTE0tMTM1TS1JbnN0cnVjdCU3QzAuMDElN0MlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAwLjAxJTdDJTBBJTBBcmVzdWx0cyUyMHNhdmVkJTIwdG8lMjBsaWdodGV2YWwtbG9ncyUwQXJ1biUyMCUyMmluc3BlY3QlMjB2aWV3JTIwLS1sb2ctZGlyJTIwbGlnaHRldmFsLWxvZ3MlMjIlMjB0byUyMHZpZXclMjB0aGUlMjByZXN1bHRz",highlighted:`<span class="hljs-title class_">Completed</span> all tasks <span class="hljs-keyword">in</span> <span class="hljs-string">&#x27;lighteval-logs&#x27;</span> successfully

|                 <span class="hljs-title class_">Model</span>                 |gpqa|<span class="hljs-symbol">gpqa:</span>diamond|
|---------------------------------------|---<span class="hljs-symbol">:|-----------</span><span class="hljs-symbol">:|</span>
|vllm/<span class="hljs-title class_">HuggingFaceTB</span>/<span class="hljs-title class_">SmolLM</span><span class="hljs-number">-135</span>M-<span class="hljs-title class_">Instruct</span>|<span class="hljs-number">0.01</span>|        <span class="hljs-number">0.01</span>|

results saved to lighteval-logs
run <span class="hljs-string">&quot;inspect view --log-dir lighteval-logs&quot;</span> to view the results`,wrap:!1}}),P=new yt({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/inspect-ai.mdx"}}),{c(){T=i("meta"),ee=a(),D=i("p"),te=a(),r(W.$$.fragment),le=a(),r(I.$$.fragment),se=a(),k=i("p"),k.textContent=Ae,ae=a(),$=i("blockquote"),$.innerHTML=Re,ne=a(),h=i("iframe"),ie=a(),G=i("p"),G.innerHTML=Ye,me=a(),r(x.$$.fragment),pe=a(),S=i("ol"),S.innerHTML=ze,re=a(),r(H.$$.fragment),Me=a(),v=i("ol"),v.innerHTML=Ee,ue=a(),r(_.$$.fragment),fe=a(),d=i("ol"),d.innerHTML=Ne,ye=a(),r(V.$$.fragment),ce=a(),U=i("ol"),U.innerHTML=Qe,oe=a(),r(A.$$.fragment),be=a(),g=i("ol"),g.innerHTML=Fe,he=a(),r(R.$$.fragment),we=a(),Z=i("ol"),Z.innerHTML=Xe,Te=a(),r(B.$$.fragment),$e=a(),C=i("ol"),C.innerHTML=qe,ve=a(),r(Y.$$.fragment),de=a(),j=i("ol"),j.innerHTML=Oe,Ue=a(),r(z.$$.fragment),ge=a(),E=i("p"),E.textContent=Pe,Ze=a(),w=i("iframe"),Ce=a(),L=i("ol"),L.innerHTML=Ke,je=a(),N=i("p"),N.textContent=et,Le=a(),r(Q.$$.fragment),Je=a(),J=i("ol"),J.innerHTML=tt,We=a(),r(F.$$.fragment),Ie=a(),r(X.$$.fragment),ke=a(),q=i("p"),q.textContent=lt,Ge=a(),r(O.$$.fragment),xe=a(),r(P.$$.fragment),Se=a(),K=i("p"),this.h()},l(e){const t=Mt("svelte-u9bgzb",document.head);T=m(t,"META",{name:!0,content:!0}),t.forEach(l),ee=n(e),D=m(e,"P",{}),_e(D).forEach(l),te=n(e),M(W.$$.fragment,e),le=n(e),M(I.$$.fragment,e),se=n(e),k=m(e,"P",{"data-svelte-h":!0}),u(k)!=="svelte-wvxpw7"&&(k.textContent=Ae),ae=n(e),$=m(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),u($)!=="svelte-13z5e58"&&($.innerHTML=Re),ne=n(e),h=m(e,"IFRAME",{src:!0,frameborder:!0,width:!0,height:!0}),_e(h).forEach(l),ie=n(e),G=m(e,"P",{"data-svelte-h":!0}),u(G)!=="svelte-1s8wmkx"&&(G.innerHTML=Ye),me=n(e),M(x.$$.fragment,e),pe=n(e),S=m(e,"OL",{"data-svelte-h":!0}),u(S)!=="svelte-1n0ds6r"&&(S.innerHTML=ze),re=n(e),M(H.$$.fragment,e),Me=n(e),v=m(e,"OL",{start:!0,"data-svelte-h":!0}),u(v)!=="svelte-158v33h"&&(v.innerHTML=Ee),ue=n(e),M(_.$$.fragment,e),fe=n(e),d=m(e,"OL",{start:!0,"data-svelte-h":!0}),u(d)!=="svelte-12sgcy5"&&(d.innerHTML=Ne),ye=n(e),M(V.$$.fragment,e),ce=n(e),U=m(e,"OL",{start:!0,"data-svelte-h":!0}),u(U)!=="svelte-h63d4v"&&(U.innerHTML=Qe),oe=n(e),M(A.$$.fragment,e),be=n(e),g=m(e,"OL",{start:!0,"data-svelte-h":!0}),u(g)!=="svelte-1lgb0oc"&&(g.innerHTML=Fe),he=n(e),M(R.$$.fragment,e),we=n(e),Z=m(e,"OL",{start:!0,"data-svelte-h":!0}),u(Z)!=="svelte-xic7d6"&&(Z.innerHTML=Xe),Te=n(e),M(B.$$.fragment,e),$e=n(e),C=m(e,"OL",{start:!0,"data-svelte-h":!0}),u(C)!=="svelte-z8xpgu"&&(C.innerHTML=qe),ve=n(e),M(Y.$$.fragment,e),de=n(e),j=m(e,"OL",{start:!0,"data-svelte-h":!0}),u(j)!=="svelte-bkjaqd"&&(j.innerHTML=Oe),Ue=n(e),M(z.$$.fragment,e),ge=n(e),E=m(e,"P",{"data-svelte-h":!0}),u(E)!=="svelte-xmt8gz"&&(E.textContent=Pe),Ze=n(e),w=m(e,"IFRAME",{src:!0,frameborder:!0,width:!0,height:!0}),_e(w).forEach(l),Ce=n(e),L=m(e,"OL",{start:!0,"data-svelte-h":!0}),u(L)!=="svelte-65smkt"&&(L.innerHTML=Ke),je=n(e),N=m(e,"P",{"data-svelte-h":!0}),u(N)!=="svelte-1hiqzh0"&&(N.textContent=et),Le=n(e),M(Q.$$.fragment,e),Je=n(e),J=m(e,"OL",{start:!0,"data-svelte-h":!0}),u(J)!=="svelte-p39luw"&&(J.innerHTML=tt),We=n(e),M(F.$$.fragment,e),Ie=n(e),M(X.$$.fragment,e),ke=n(e),q=m(e,"P",{"data-svelte-h":!0}),u(q)!=="svelte-g5np0a"&&(q.textContent=lt),Ge=n(e),M(O.$$.fragment,e),xe=n(e),M(P.$$.fragment,e),Se=n(e),K=m(e,"P",{}),_e(K).forEach(l),this.h()},h(){p(T,"name","hf:doc:metadata"),p(T,"content",ot),p($,"class","warning"),st(h.src,Be="https://openevals-open-benchmark-index.hf.space")||p(h,"src",Be),p(h,"frameborder","0"),p(h,"width","850"),p(h,"height","450"),p(v,"start","2"),p(d,"start","3"),p(U,"start","4"),p(g,"start","5"),p(Z,"start","6"),p(C,"start","7"),p(j,"start","8"),st(w.src,De="https://openevals-evals.static.hf.space")||p(w,"src",De),p(w,"frameborder","0"),p(w,"width","850"),p(w,"height","450"),p(L,"start","9"),p(J,"start","10")},m(e,t){ut(document.head,T),s(e,ee,t),s(e,D,t),s(e,te,t),f(W,e,t),s(e,le,t),f(I,e,t),s(e,se,t),s(e,k,t),s(e,ae,t),s(e,$,t),s(e,ne,t),s(e,h,t),s(e,ie,t),s(e,G,t),s(e,me,t),f(x,e,t),s(e,pe,t),s(e,S,t),s(e,re,t),f(H,e,t),s(e,Me,t),s(e,v,t),s(e,ue,t),f(_,e,t),s(e,fe,t),s(e,d,t),s(e,ye,t),f(V,e,t),s(e,ce,t),s(e,U,t),s(e,oe,t),f(A,e,t),s(e,be,t),s(e,g,t),s(e,he,t),f(R,e,t),s(e,we,t),s(e,Z,t),s(e,Te,t),f(B,e,t),s(e,$e,t),s(e,C,t),s(e,ve,t),f(Y,e,t),s(e,de,t),s(e,j,t),s(e,Ue,t),f(z,e,t),s(e,ge,t),s(e,E,t),s(e,Ze,t),s(e,w,t),s(e,Ce,t),s(e,L,t),s(e,je,t),s(e,N,t),s(e,Le,t),f(Q,e,t),s(e,Je,t),s(e,J,t),s(e,We,t),f(F,e,t),s(e,Ie,t),f(X,e,t),s(e,ke,t),s(e,q,t),s(e,Ge,t),f(O,e,t),s(e,xe,t),f(P,e,t),s(e,Se,t),s(e,K,t),He=!0},p:it,i(e){He||(y(W.$$.fragment,e),y(I.$$.fragment,e),y(x.$$.fragment,e),y(H.$$.fragment,e),y(_.$$.fragment,e),y(V.$$.fragment,e),y(A.$$.fragment,e),y(R.$$.fragment,e),y(B.$$.fragment,e),y(Y.$$.fragment,e),y(z.$$.fragment,e),y(Q.$$.fragment,e),y(F.$$.fragment,e),y(X.$$.fragment,e),y(O.$$.fragment,e),y(P.$$.fragment,e),He=!0)},o(e){c(W.$$.fragment,e),c(I.$$.fragment,e),c(x.$$.fragment,e),c(H.$$.fragment,e),c(_.$$.fragment,e),c(V.$$.fragment,e),c(A.$$.fragment,e),c(R.$$.fragment,e),c(B.$$.fragment,e),c(Y.$$.fragment,e),c(z.$$.fragment,e),c(Q.$$.fragment,e),c(F.$$.fragment,e),c(X.$$.fragment,e),c(O.$$.fragment,e),c(P.$$.fragment,e),He=!1},d(e){e&&(l(ee),l(D),l(te),l(le),l(se),l(k),l(ae),l($),l(ne),l(h),l(ie),l(G),l(me),l(pe),l(S),l(re),l(Me),l(v),l(ue),l(fe),l(d),l(ye),l(ce),l(U),l(oe),l(be),l(g),l(he),l(we),l(Z),l(Te),l($e),l(C),l(ve),l(de),l(j),l(Ue),l(ge),l(E),l(Ze),l(w),l(Ce),l(L),l(je),l(N),l(Le),l(Je),l(J),l(We),l(Ie),l(ke),l(q),l(Ge),l(xe),l(Se),l(K)),l(T),o(W,e),o(I,e),o(x,e),o(H,e),o(_,e),o(V,e),o(A,e),o(R,e),o(B,e),o(Y,e),o(z,e),o(Q,e),o(F,e),o(X,e),o(O,e),o(P,e)}}}const ot='{"title":"Evaluate your model with Inspect-AI","local":"evaluate-your-model-with-inspect-ai","sections":[{"title":"Examples","local":"examples","sections":[],"depth":3}],"depth":1}';function bt(Ve){return mt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class vt extends pt{constructor(T){super(),rt(this,T,bt,ct,nt,{})}}export{vt as component};
