import{s as ge,n as ue,o as de}from"../chunks/scheduler.a045fce0.js";import{S as fe,i as he,e as i,s as a,c as M,h as ce,a as o,d as n,b as s,f as me,g as P,j as r,k as pe,l as ve,m as l,n as S,t as C,o as H,p as G}from"../chunks/index.c7f31426.js";import{H as X,E as $e}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.8ed610dd.js";function we(Y){let m,z,y,U,p,E,g,Z=`SGLang is a fast serving framework for large language models and vision language models. It’s very similar to TGI and vLLM and comes
with production ready features.`,q,u,ee="The core features include:",F,d,te=`<li><p><strong>Fast Backend Runtime</strong>:</p> <ul><li>efficient serving with RadixAttention for prefix caching</li> <li>zero-overhead CPU scheduler</li> <li>continuous batching, paged attention, tensor parallelism and pipeline parallelism,</li> <li>expert parallelism, structured outputs, chunked prefill, quantization (FP8/INT4/AWQ/GPTQ), and multi-lora batching</li></ul></li> <li><p><strong>Extensive Model Support</strong>: Supports a wide range of generative models (Llama, Gemma, Mistral, Qwen, DeepSeek, LLaVA, etc.),
embedding models (e5-mistral, gte, mcdse) and reward models (Skywork), with easy extensibility for integrating new models.</p></li>`,I,f,R,h,ne='<img src="https://raw.githubusercontent.com/huggingface/hf-endpoints-documentation/main/assets/sglang/sglang.png" alt="sglang"/>',A,c,le=`<li><strong>Max Running Request</strong>: the max number of concurrent requests</li> <li><strong>Max Prefill Tokens</strong> (per batch): the maximum number of tokens that can be processed in a single prefill operation. This controls the batch size for the prefill phase and helps manage memory usage during prompt processing.</li> <li><strong>Chunked prefill size</strong>: sets how many tokens are processed at once during the prefill phase. If a prompt is longer than this value,
it will be split into smaller chunks and processed sequentially to avoid out-of-memory errors during prefill with long prompts.
For example, setting —chunked-prefill-size 4096 means each chunk will have up to 4096 tokens processed at a time. Setting this to -1
means disabling chunked prefill.</li> <li><strong>Tensor Parallel Size</strong>: the number of GPUs to use for tensor parallelism. This enables model sharding across multiple GPUs
to handle larger models that don’t fit on a single GPU. For example, setting this to 2 will split the model across 2 GPUs.</li> <li><strong>KV Cache DType</strong>: the data type used for storing the key-value cache during generation. Options include “auto”, “fp8_e5m2”,
and “fp8_e4m3”. Using lower precision types can reduce memory usage but may slightly impact generation quality.</li>`,Q,v,ae=`For more advanced configuration you can pass any of the <a href="https://docs.sglang.ai/backend/server_arguments.html" rel="nofollow">Server Arguments that SGlang supports</a>
as container arguments. For example changing the <code>schedule-policy</code> to <code>lpm</code> would look like this:`,W,$,se='<img src="https://raw.githubusercontent.com/huggingface/hf-endpoints-documentation/main/assets/sglang/sglang-advanced.png" alt="sglang-advanced"/>',D,w,O,b,ie=`SGlang has wide support for large language models, multimodal language models, embedding models and more. We recommend reading the
<a href="https://docs.sglang.ai/supported_models/generative_models.html" rel="nofollow">supported models</a> section in the SGLang documentation for a full list.`,V,_,oe=`In the Inference Endpoints UI, by default, any model on the Hugging Face Hub that has a <code>transformers</code> tag, can be deployed with SGLang.
This is because SGLang <a href="https://docs.sglang.ai/supported_models/transformers_fallback.html#transformers-fallback-in-sglang" rel="nofollow">implements a fallback</a> to use transformers
if SGLang doesn’t have their own implementation of a model.`,j,L,B,x,re='We also recommend reading the <a href="https://docs.sglang.ai/" rel="nofollow">SGLang documentation</a> for more in-depth information.',K,T,N,k,J;return p=new X({props:{title:"SGLang",local:"sglang",headingTag:"h1"}}),f=new X({props:{title:"Configuration",local:"configuration",headingTag:"h2"}}),w=new X({props:{title:"Supported models",local:"supported-models",headingTag:"h2"}}),L=new X({props:{title:"References",local:"references",headingTag:"h2"}}),T=new $e({props:{source:"https://github.com/huggingface/hf-endpoints-documentation/blob/main/docs/source/engines/sglang.md"}}),{c(){m=i("meta"),z=a(),y=i("p"),U=a(),M(p.$$.fragment),E=a(),g=i("p"),g.textContent=Z,q=a(),u=i("p"),u.textContent=ee,F=a(),d=i("ul"),d.innerHTML=te,I=a(),M(f.$$.fragment),R=a(),h=i("p"),h.innerHTML=ne,A=a(),c=i("ul"),c.innerHTML=le,Q=a(),v=i("p"),v.innerHTML=ae,W=a(),$=i("p"),$.innerHTML=se,D=a(),M(w.$$.fragment),O=a(),b=i("p"),b.innerHTML=ie,V=a(),_=i("p"),_.innerHTML=oe,j=a(),M(L.$$.fragment),B=a(),x=i("p"),x.innerHTML=re,K=a(),M(T.$$.fragment),N=a(),k=i("p"),this.h()},l(e){const t=ce("svelte-u9bgzb",document.head);m=o(t,"META",{name:!0,content:!0}),t.forEach(n),z=s(e),y=o(e,"P",{}),me(y).forEach(n),U=s(e),P(p.$$.fragment,e),E=s(e),g=o(e,"P",{"data-svelte-h":!0}),r(g)!=="svelte-i4zzvk"&&(g.textContent=Z),q=s(e),u=o(e,"P",{"data-svelte-h":!0}),r(u)!=="svelte-1h7t0xv"&&(u.textContent=ee),F=s(e),d=o(e,"UL",{"data-svelte-h":!0}),r(d)!=="svelte-i8wyr8"&&(d.innerHTML=te),I=s(e),P(f.$$.fragment,e),R=s(e),h=o(e,"P",{"data-svelte-h":!0}),r(h)!=="svelte-mw3pr2"&&(h.innerHTML=ne),A=s(e),c=o(e,"UL",{"data-svelte-h":!0}),r(c)!=="svelte-v7hqhg"&&(c.innerHTML=le),Q=s(e),v=o(e,"P",{"data-svelte-h":!0}),r(v)!=="svelte-bu8esv"&&(v.innerHTML=ae),W=s(e),$=o(e,"P",{"data-svelte-h":!0}),r($)!=="svelte-zlixhk"&&($.innerHTML=se),D=s(e),P(w.$$.fragment,e),O=s(e),b=o(e,"P",{"data-svelte-h":!0}),r(b)!=="svelte-1b1idos"&&(b.innerHTML=ie),V=s(e),_=o(e,"P",{"data-svelte-h":!0}),r(_)!=="svelte-76vkhp"&&(_.innerHTML=oe),j=s(e),P(L.$$.fragment,e),B=s(e),x=o(e,"P",{"data-svelte-h":!0}),r(x)!=="svelte-1yhkg23"&&(x.innerHTML=re),K=s(e),P(T.$$.fragment,e),N=s(e),k=o(e,"P",{}),me(k).forEach(n),this.h()},h(){pe(m,"name","hf:doc:metadata"),pe(m,"content",be)},m(e,t){ve(document.head,m),l(e,z,t),l(e,y,t),l(e,U,t),S(p,e,t),l(e,E,t),l(e,g,t),l(e,q,t),l(e,u,t),l(e,F,t),l(e,d,t),l(e,I,t),S(f,e,t),l(e,R,t),l(e,h,t),l(e,A,t),l(e,c,t),l(e,Q,t),l(e,v,t),l(e,W,t),l(e,$,t),l(e,D,t),S(w,e,t),l(e,O,t),l(e,b,t),l(e,V,t),l(e,_,t),l(e,j,t),S(L,e,t),l(e,B,t),l(e,x,t),l(e,K,t),S(T,e,t),l(e,N,t),l(e,k,t),J=!0},p:ue,i(e){J||(C(p.$$.fragment,e),C(f.$$.fragment,e),C(w.$$.fragment,e),C(L.$$.fragment,e),C(T.$$.fragment,e),J=!0)},o(e){H(p.$$.fragment,e),H(f.$$.fragment,e),H(w.$$.fragment,e),H(L.$$.fragment,e),H(T.$$.fragment,e),J=!1},d(e){e&&(n(z),n(y),n(U),n(E),n(g),n(q),n(u),n(F),n(d),n(I),n(R),n(h),n(A),n(c),n(Q),n(v),n(W),n($),n(D),n(O),n(b),n(V),n(_),n(j),n(B),n(x),n(K),n(N),n(k)),n(m),G(p,e),G(f,e),G(w,e),G(L,e),G(T,e)}}}const be='{"title":"SGLang","local":"sglang","sections":[{"title":"Configuration","local":"configuration","sections":[],"depth":2},{"title":"Supported models","local":"supported-models","sections":[],"depth":2},{"title":"References","local":"references","sections":[],"depth":2}],"depth":1}';function _e(Y){return de(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ye extends fe{constructor(m){super(),he(this,m,_e,we,ge,{})}}export{ye as component};
