import{s as Ne,n as Ve,o as Le}from"../chunks/scheduler.3a17fb72.js";import{S as Qe,i as Fe,e as M,s,c as i,h as Ye,a as u,d as t,b as n,f as ze,g as p,j as f,k as Ie,l as qe,m as a,n as r,t as m,o,p as c}from"../chunks/index.093f8863.js";import{C as Xe}from"../chunks/CopyLLMTxtMenu.ccc246bc.js";import{C as X}from"../chunks/CodeBlock.b126c7de.js";import{H as d,E as Pe}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.ff8099de.js";function Re(Ce){let g,P,Y,R,y,D,$,O,w,Be=`Lighteval allows you to use SGLang as a backend, providing significant speedups for model evaluation.
To use SGLang, simply change the <code>model_args</code> to reflect the arguments you want to pass to SGLang.`,K,j,ee,J,le,U,te,b,Ge=`SGLang can distribute the model across multiple GPUs using data parallelism and tensor parallelism.
You can choose the parallelism method by setting the appropriate parameters in the <code>model_args</code>.`,ae,_,se,I,ke="For example, if you have 4 GPUs, you can split the model across them using tensor parallelism with <code>tp_size</code>:",ne,C,ie,B,pe,G,ve="If your model fits on a single GPU, you can use data parallelism with <code>dp_size</code> to speed up the evaluation:",re,k,me,v,oe,S,Se=`For more advanced configurations, you can use a YAML configuration file for the model.
An example configuration file is shown below and can be found at <code>examples/model_configs/sglang_model_config.yaml</code>.`,ce,Z,Me,T,Ze='<p>Documentation for SGLang server arguments can be found <a href="https://docs.sglang.ai/backend/server_arguments.html" rel="nofollow">here</a></p>',ue,A,fe,h,Ae=`<p>In case of out-of-memory (OOM) issues, you might need to reduce the context size of the
model as well as reduce the <code>mem_fraction_static</code> and <code>chunked_prefill_size</code> parameters.</p>`,de,x,ge,W,Te,E,xe="<li><code>mem_fraction_static</code>: Fraction of GPU memory to allocate for static tensors (default: 0.8)</li> <li><code>chunked_prefill_size</code>: Size of chunks for prefill operations (default: 4096)</li> <li><code>context_length</code>: Maximum context length for the model</li> <li><code>kv_cache_dtype</code>: Data type for key-value cache</li>",he,H,ye,z,We="<li><code>tp_size</code>: Number of GPUs for tensor parallelism</li> <li><code>dp_size</code>: Number of GPUs for data parallelism</li>",$e,N,we,V,Ee="<li><code>dtype</code>: Data type for model weights (“auto”, “float16”, “bfloat16”, etc.)</li> <li><code>device</code>: Device to run the model on (“cuda”, “cpu”)</li> <li><code>trust_remote_code</code>: Whether to trust remote code from the model</li> <li><code>skip_tokenizer_init</code>: Skip tokenizer initialization for faster startup</li>",je,L,Je,Q,He="<li><code>temperature</code>: Controls randomness in generation (0.0 = deterministic, 1.0 = random)</li> <li><code>top_p</code>: Nucleus sampling parameter</li> <li><code>top_k</code>: Top-k sampling parameter</li> <li><code>max_new_tokens</code>: Maximum number of tokens to generate</li> <li><code>repetition_penalty</code>: Penalty for repeating tokens</li> <li><code>presence_penalty</code>: Penalty for token presence</li> <li><code>frequency_penalty</code>: Penalty for token frequency</li>",Ue,F,be,q,_e;return y=new Xe({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),$=new d({props:{title:"Using SGLang as Backend",local:"using-sglang-as-backend",headingTag:"h1"}}),j=new d({props:{title:"Basic Usage",local:"basic-usage",headingTag:"h2"}}),J=new X({props:{code:"bGlnaHRldmFsJTIwc2dsYW5nJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybW9kZWxfbmFtZSUzREh1Z2dpbmdGYWNlSDQlMkZ6ZXBoeXItN2ItYmV0YSUyQ2R0eXBlJTNEZmxvYXQxNiUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMCUyMmxlYWRlcmJvYXJkJTdDdHJ1dGhmdWxxYSUzQW1jJTdDMCUyMg==",highlighted:`lighteval sglang \\
    <span class="hljs-string">&quot;model_name=HuggingFaceH4/zephyr-7b-beta,dtype=float16&quot;</span> \\
    <span class="hljs-string">&quot;leaderboard|truthfulqa:mc|0&quot;</span>`,wrap:!1}}),U=new d({props:{title:"Parallelism Options",local:"parallelism-options",headingTag:"h2"}}),_=new d({props:{title:"Tensor Parallelism",local:"tensor-parallelism",headingTag:"h3"}}),C=new X({props:{code:"bGlnaHRldmFsJTIwc2dsYW5nJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybW9kZWxfbmFtZSUzREh1Z2dpbmdGYWNlSDQlMkZ6ZXBoeXItN2ItYmV0YSUyQ2R0eXBlJTNEZmxvYXQxNiUyQ3RwX3NpemUlM0Q0JTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybGVhZGVyYm9hcmQlN0N0cnV0aGZ1bHFhJTNBbWMlN0MwJTIy",highlighted:`lighteval sglang \\
    <span class="hljs-string">&quot;model_name=HuggingFaceH4/zephyr-7b-beta,dtype=float16,tp_size=4&quot;</span> \\
    <span class="hljs-string">&quot;leaderboard|truthfulqa:mc|0&quot;</span>`,wrap:!1}}),B=new d({props:{title:"Data Parallelism",local:"data-parallelism",headingTag:"h3"}}),k=new X({props:{code:"bGlnaHRldmFsJTIwc2dsYW5nJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybW9kZWxfbmFtZSUzREh1Z2dpbmdGYWNlSDQlMkZ6ZXBoeXItN2ItYmV0YSUyQ2R0eXBlJTNEZmxvYXQxNiUyQ2RwX3NpemUlM0Q0JTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybGVhZGVyYm9hcmQlN0N0cnV0aGZ1bHFhJTNBbWMlN0MwJTIy",highlighted:`lighteval sglang \\
    <span class="hljs-string">&quot;model_name=HuggingFaceH4/zephyr-7b-beta,dtype=float16,dp_size=4&quot;</span> \\
    <span class="hljs-string">&quot;leaderboard|truthfulqa:mc|0&quot;</span>`,wrap:!1}}),v=new d({props:{title:"Using a Configuration File",local:"using-a-configuration-file",headingTag:"h2"}}),Z=new X({props:{code:"bGlnaHRldmFsJTIwc2dsYW5nJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIyZXhhbXBsZXMlMkZtb2RlbF9jb25maWdzJTJGc2dsYW5nX21vZGVsX2NvbmZpZy55YW1sJTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybGVhZGVyYm9hcmQlN0N0cnV0aGZ1bHFhJTNBbWMlN0MwJTIy",highlighted:`lighteval sglang \\
    <span class="hljs-string">&quot;examples/model_configs/sglang_model_config.yaml&quot;</span> \\
    <span class="hljs-string">&quot;leaderboard|truthfulqa:mc|0&quot;</span>`,wrap:!1}}),A=new X({props:{code:"bW9kZWxfcGFyYW1ldGVycyUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlM0ElMjAlMjJIdWdnaW5nRmFjZVRCJTJGU21vbExNLTEuN0ItSW5zdHJ1Y3QlMjIlMEElMjAlMjAlMjAlMjBkdHlwZSUzQSUyMCUyMmF1dG8lMjIlMEElMjAlMjAlMjAlMjB0cF9zaXplJTNBJTIwMSUwQSUyMCUyMCUyMCUyMGRwX3NpemUlM0ElMjAxJTBBJTIwJTIwJTIwJTIwY29udGV4dF9sZW5ndGglM0ElMjBudWxsJTBBJTIwJTIwJTIwJTIwcmFuZG9tX3NlZWQlM0ElMjAxJTBBJTIwJTIwJTIwJTIwdHJ1c3RfcmVtb3RlX2NvZGUlM0ElMjBGYWxzZSUwQSUyMCUyMCUyMCUyMGRldmljZSUzQSUyMCUyMmN1ZGElMjIlMEElMjAlMjAlMjAlMjBza2lwX3Rva2VuaXplcl9pbml0JTNBJTIwRmFsc2UlMEElMjAlMjAlMjAlMjBrdl9jYWNoZV9kdHlwZSUzQSUyMCUyMmF1dG8lMjIlMEElMjAlMjAlMjAlMjBhZGRfc3BlY2lhbF90b2tlbnMlM0ElMjBUcnVlJTBBJTIwJTIwJTIwJTIwcGFpcndpc2VfdG9rZW5pemF0aW9uJTNBJTIwRmFsc2UlMEElMjAlMjAlMjAlMjBzYW1wbGluZ19iYWNrZW5kJTNBJTIwbnVsbCUwQSUyMCUyMCUyMCUyMGF0dGVudGlvbl9iYWNrZW5kJTNBJTIwbnVsbCUwQSUyMCUyMCUyMCUyMG1lbV9mcmFjdGlvbl9zdGF0aWMlM0ElMjAwLjglMEElMjAlMjAlMjAlMjBjaHVua2VkX3ByZWZpbGxfc2l6ZSUzQSUyMDQwOTYlMEElMjAlMjAlMjAlMjBnZW5lcmF0aW9uX3BhcmFtZXRlcnMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjBtYXhfbmV3X3Rva2VucyUzQSUyMDEwMjQlMEElMjAlMjAlMjAlMjAlMjAlMjBtaW5fbmV3X3Rva2VucyUzQSUyMDAlMEElMjAlMjAlMjAlMjAlMjAlMjB0ZW1wZXJhdHVyZSUzQSUyMDEuMCUwQSUyMCUyMCUyMCUyMCUyMCUyMHRvcF9rJTNBJTIwNTAlMEElMjAlMjAlMjAlMjAlMjAlMjBtaW5fcCUzQSUyMDAuMCUwQSUyMCUyMCUyMCUyMCUyMCUyMHRvcF9wJTNBJTIwMS4wJTBBJTIwJTIwJTIwJTIwJTIwJTIwcHJlc2VuY2VfcGVuYWx0eSUzQSUyMDAuMCUwQSUyMCUyMCUyMCUyMCUyMCUyMHJlcGV0aXRpb25fcGVuYWx0eSUzQSUyMDEuMCUwQSUyMCUyMCUyMCUyMCUyMCUyMGZyZXF1ZW5jeV9wZW5hbHR5JTNBJTIwMC4w",highlighted:`<span class="hljs-attr">model_parameters:</span>
    <span class="hljs-attr">model_name:</span> <span class="hljs-string">&quot;HuggingFaceTB/SmolLM-1.7B-Instruct&quot;</span>
    <span class="hljs-attr">dtype:</span> <span class="hljs-string">&quot;auto&quot;</span>
    <span class="hljs-attr">tp_size:</span> <span class="hljs-number">1</span>
    <span class="hljs-attr">dp_size:</span> <span class="hljs-number">1</span>
    <span class="hljs-attr">context_length:</span> <span class="hljs-literal">null</span>
    <span class="hljs-attr">random_seed:</span> <span class="hljs-number">1</span>
    <span class="hljs-attr">trust_remote_code:</span> <span class="hljs-literal">False</span>
    <span class="hljs-attr">device:</span> <span class="hljs-string">&quot;cuda&quot;</span>
    <span class="hljs-attr">skip_tokenizer_init:</span> <span class="hljs-literal">False</span>
    <span class="hljs-attr">kv_cache_dtype:</span> <span class="hljs-string">&quot;auto&quot;</span>
    <span class="hljs-attr">add_special_tokens:</span> <span class="hljs-literal">True</span>
    <span class="hljs-attr">pairwise_tokenization:</span> <span class="hljs-literal">False</span>
    <span class="hljs-attr">sampling_backend:</span> <span class="hljs-literal">null</span>
    <span class="hljs-attr">attention_backend:</span> <span class="hljs-literal">null</span>
    <span class="hljs-attr">mem_fraction_static:</span> <span class="hljs-number">0.8</span>
    <span class="hljs-attr">chunked_prefill_size:</span> <span class="hljs-number">4096</span>
    <span class="hljs-attr">generation_parameters:</span>
      <span class="hljs-attr">max_new_tokens:</span> <span class="hljs-number">1024</span>
      <span class="hljs-attr">min_new_tokens:</span> <span class="hljs-number">0</span>
      <span class="hljs-attr">temperature:</span> <span class="hljs-number">1.0</span>
      <span class="hljs-attr">top_k:</span> <span class="hljs-number">50</span>
      <span class="hljs-attr">min_p:</span> <span class="hljs-number">0.0</span>
      <span class="hljs-attr">top_p:</span> <span class="hljs-number">1.0</span>
      <span class="hljs-attr">presence_penalty:</span> <span class="hljs-number">0.0</span>
      <span class="hljs-attr">repetition_penalty:</span> <span class="hljs-number">1.0</span>
      <span class="hljs-attr">frequency_penalty:</span> <span class="hljs-number">0.0</span>`,wrap:!1}}),x=new d({props:{title:"Key SGLang Parameters",local:"key-sglang-parameters",headingTag:"h2"}}),W=new d({props:{title:"Memory Management",local:"memory-management",headingTag:"h3"}}),H=new d({props:{title:"Parallelism Settings",local:"parallelism-settings",headingTag:"h3"}}),N=new d({props:{title:"Model Configuration",local:"model-configuration",headingTag:"h3"}}),L=new d({props:{title:"Generation Parameters",local:"generation-parameters",headingTag:"h3"}}),F=new Pe({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/use-sglang-as-backend.mdx"}}),{c(){g=M("meta"),P=s(),Y=M("p"),R=s(),i(y.$$.fragment),D=s(),i($.$$.fragment),O=s(),w=M("p"),w.innerHTML=Be,K=s(),i(j.$$.fragment),ee=s(),i(J.$$.fragment),le=s(),i(U.$$.fragment),te=s(),b=M("p"),b.innerHTML=Ge,ae=s(),i(_.$$.fragment),se=s(),I=M("p"),I.innerHTML=ke,ne=s(),i(C.$$.fragment),ie=s(),i(B.$$.fragment),pe=s(),G=M("p"),G.innerHTML=ve,re=s(),i(k.$$.fragment),me=s(),i(v.$$.fragment),oe=s(),S=M("p"),S.innerHTML=Se,ce=s(),i(Z.$$.fragment),Me=s(),T=M("blockquote"),T.innerHTML=Ze,ue=s(),i(A.$$.fragment),fe=s(),h=M("blockquote"),h.innerHTML=Ae,de=s(),i(x.$$.fragment),ge=s(),i(W.$$.fragment),Te=s(),E=M("ul"),E.innerHTML=xe,he=s(),i(H.$$.fragment),ye=s(),z=M("ul"),z.innerHTML=We,$e=s(),i(N.$$.fragment),we=s(),V=M("ul"),V.innerHTML=Ee,je=s(),i(L.$$.fragment),Je=s(),Q=M("ul"),Q.innerHTML=He,Ue=s(),i(F.$$.fragment),be=s(),q=M("p"),this.h()},l(e){const l=Ye("svelte-u9bgzb",document.head);g=u(l,"META",{name:!0,content:!0}),l.forEach(t),P=n(e),Y=u(e,"P",{}),ze(Y).forEach(t),R=n(e),p(y.$$.fragment,e),D=n(e),p($.$$.fragment,e),O=n(e),w=u(e,"P",{"data-svelte-h":!0}),f(w)!=="svelte-1bw11uc"&&(w.innerHTML=Be),K=n(e),p(j.$$.fragment,e),ee=n(e),p(J.$$.fragment,e),le=n(e),p(U.$$.fragment,e),te=n(e),b=u(e,"P",{"data-svelte-h":!0}),f(b)!=="svelte-10zaw8k"&&(b.innerHTML=Ge),ae=n(e),p(_.$$.fragment,e),se=n(e),I=u(e,"P",{"data-svelte-h":!0}),f(I)!=="svelte-odna94"&&(I.innerHTML=ke),ne=n(e),p(C.$$.fragment,e),ie=n(e),p(B.$$.fragment,e),pe=n(e),G=u(e,"P",{"data-svelte-h":!0}),f(G)!=="svelte-15m6api"&&(G.innerHTML=ve),re=n(e),p(k.$$.fragment,e),me=n(e),p(v.$$.fragment,e),oe=n(e),S=u(e,"P",{"data-svelte-h":!0}),f(S)!=="svelte-560e5r"&&(S.innerHTML=Se),ce=n(e),p(Z.$$.fragment,e),Me=n(e),T=u(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),f(T)!=="svelte-b8ev60"&&(T.innerHTML=Ze),ue=n(e),p(A.$$.fragment,e),fe=n(e),h=u(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),f(h)!=="svelte-loo1zp"&&(h.innerHTML=Ae),de=n(e),p(x.$$.fragment,e),ge=n(e),p(W.$$.fragment,e),Te=n(e),E=u(e,"UL",{"data-svelte-h":!0}),f(E)!=="svelte-h1z7e"&&(E.innerHTML=xe),he=n(e),p(H.$$.fragment,e),ye=n(e),z=u(e,"UL",{"data-svelte-h":!0}),f(z)!=="svelte-1k89zlt"&&(z.innerHTML=We),$e=n(e),p(N.$$.fragment,e),we=n(e),V=u(e,"UL",{"data-svelte-h":!0}),f(V)!=="svelte-19ebxti"&&(V.innerHTML=Ee),je=n(e),p(L.$$.fragment,e),Je=n(e),Q=u(e,"UL",{"data-svelte-h":!0}),f(Q)!=="svelte-1s61mt7"&&(Q.innerHTML=He),Ue=n(e),p(F.$$.fragment,e),be=n(e),q=u(e,"P",{}),ze(q).forEach(t),this.h()},h(){Ie(g,"name","hf:doc:metadata"),Ie(g,"content",De),Ie(T,"class","tip"),Ie(h,"class","warning")},m(e,l){qe(document.head,g),a(e,P,l),a(e,Y,l),a(e,R,l),r(y,e,l),a(e,D,l),r($,e,l),a(e,O,l),a(e,w,l),a(e,K,l),r(j,e,l),a(e,ee,l),r(J,e,l),a(e,le,l),r(U,e,l),a(e,te,l),a(e,b,l),a(e,ae,l),r(_,e,l),a(e,se,l),a(e,I,l),a(e,ne,l),r(C,e,l),a(e,ie,l),r(B,e,l),a(e,pe,l),a(e,G,l),a(e,re,l),r(k,e,l),a(e,me,l),r(v,e,l),a(e,oe,l),a(e,S,l),a(e,ce,l),r(Z,e,l),a(e,Me,l),a(e,T,l),a(e,ue,l),r(A,e,l),a(e,fe,l),a(e,h,l),a(e,de,l),r(x,e,l),a(e,ge,l),r(W,e,l),a(e,Te,l),a(e,E,l),a(e,he,l),r(H,e,l),a(e,ye,l),a(e,z,l),a(e,$e,l),r(N,e,l),a(e,we,l),a(e,V,l),a(e,je,l),r(L,e,l),a(e,Je,l),a(e,Q,l),a(e,Ue,l),r(F,e,l),a(e,be,l),a(e,q,l),_e=!0},p:Ve,i(e){_e||(m(y.$$.fragment,e),m($.$$.fragment,e),m(j.$$.fragment,e),m(J.$$.fragment,e),m(U.$$.fragment,e),m(_.$$.fragment,e),m(C.$$.fragment,e),m(B.$$.fragment,e),m(k.$$.fragment,e),m(v.$$.fragment,e),m(Z.$$.fragment,e),m(A.$$.fragment,e),m(x.$$.fragment,e),m(W.$$.fragment,e),m(H.$$.fragment,e),m(N.$$.fragment,e),m(L.$$.fragment,e),m(F.$$.fragment,e),_e=!0)},o(e){o(y.$$.fragment,e),o($.$$.fragment,e),o(j.$$.fragment,e),o(J.$$.fragment,e),o(U.$$.fragment,e),o(_.$$.fragment,e),o(C.$$.fragment,e),o(B.$$.fragment,e),o(k.$$.fragment,e),o(v.$$.fragment,e),o(Z.$$.fragment,e),o(A.$$.fragment,e),o(x.$$.fragment,e),o(W.$$.fragment,e),o(H.$$.fragment,e),o(N.$$.fragment,e),o(L.$$.fragment,e),o(F.$$.fragment,e),_e=!1},d(e){e&&(t(P),t(Y),t(R),t(D),t(O),t(w),t(K),t(ee),t(le),t(te),t(b),t(ae),t(se),t(I),t(ne),t(ie),t(pe),t(G),t(re),t(me),t(oe),t(S),t(ce),t(Me),t(T),t(ue),t(fe),t(h),t(de),t(ge),t(Te),t(E),t(he),t(ye),t(z),t($e),t(we),t(V),t(je),t(Je),t(Q),t(Ue),t(be),t(q)),t(g),c(y,e),c($,e),c(j,e),c(J,e),c(U,e),c(_,e),c(C,e),c(B,e),c(k,e),c(v,e),c(Z,e),c(A,e),c(x,e),c(W,e),c(H,e),c(N,e),c(L,e),c(F,e)}}}const De='{"title":"Using SGLang as Backend","local":"using-sglang-as-backend","sections":[{"title":"Basic Usage","local":"basic-usage","sections":[],"depth":2},{"title":"Parallelism Options","local":"parallelism-options","sections":[{"title":"Tensor Parallelism","local":"tensor-parallelism","sections":[],"depth":3},{"title":"Data Parallelism","local":"data-parallelism","sections":[],"depth":3}],"depth":2},{"title":"Using a Configuration File","local":"using-a-configuration-file","sections":[],"depth":2},{"title":"Key SGLang Parameters","local":"key-sglang-parameters","sections":[{"title":"Memory Management","local":"memory-management","sections":[],"depth":3},{"title":"Parallelism Settings","local":"parallelism-settings","sections":[],"depth":3},{"title":"Model Configuration","local":"model-configuration","sections":[],"depth":3},{"title":"Generation Parameters","local":"generation-parameters","sections":[],"depth":3}],"depth":2}],"depth":1}';function Oe(Ce){return Le(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class sl extends Qe{constructor(g){super(),Fe(this,g,Oe,Re,Ne,{})}}export{sl as component};
