import{s as K,n as tt,o as et}from"../chunks/scheduler.d6170356.js";import{S as lt,i as at,g as o,s as n,r as g,A as nt,h as p,f as l,c as s,j as D,u as G,x as T,k as O,y as st,a,v as C,d as v,t as I,w as j}from"../chunks/index.fcd4cc08.js";import{C as R}from"../chunks/CodeBlock.7b16bdef.js";import{H as it,E as ot}from"../chunks/EditOnGithub.da2b595c.js";function pt(A){let i,$,J,L,c,S,m,Z='Llama.cpp allows you to download and run inference on a GGUF simply by providing a path to the Hugging Face repo path and the file name. llama.cpp download the model checkpoint and automatically caches it. The location of the cache is defined by <code>LLAMA_CACHE</code> environment variable, read more about it <a href="https://github.com/ggerganov/llama.cpp/pull/7826" rel="nofollow">here</a>:',B,u,Q,r,k="Note: You can remove <code>-cnv</code> to run the CLI in chat completion mode.",W,y,z="Additionally, you can invoke an OpenAI spec chat completions endpoint directly using the llama.cpp server:",x,h,q,M,F="After running the server you can simply utilise the endpoint as below:",_,U,E,d,X="Replace <code>--hf-repo</code> with any valid Hugging Face hub repo name and <code>--hf-file</code> with the GGUF file name in the hub repo - off you go! 🦙",Y,f,P="Note: Remember to <code>build</code> llama.cpp with <code>LLAMA_CURL=1</code> :)",N,w,V,b,H;return c=new it({props:{title:"GGUF usage with llama.cpp",local:"gguf-usage-with-llamacpp",headingTag:"h1"}}),u=new R({props:{code:"LiUyRmxsYW1hLWNsaSUwQSUyMCUyMC0taGYtcmVwbyUyMGxtc3R1ZGlvLWNvbW11bml0eSUyRk1ldGEtTGxhbWEtMy04Qi1JbnN0cnVjdC1HR1VGJTIwJTVDJTBBJTIwJTIwLS1oZi1maWxlJTIwTWV0YS1MbGFtYS0zLThCLUluc3RydWN0LVE4XzAuZ2d1ZiUyMCU1QyUwQSUyMCUyMC1wJTIwJTIyWW91JTIwYXJlJTIwYSUyMGhlbHBmdWwlMjBhc3Npc3RhbnQlMjIlMjAtY252",highlighted:`./llama-cli
  --hf-repo lmstudio-community/Meta-Llama-3-8B-Instruct-GGUF \\
  --hf-file Meta-Llama-3-8B-Instruct-Q8_0.gguf \\
  -p <span class="hljs-string">&quot;You are a helpful assistant&quot;</span> -cnv`,wrap:!1}}),h=new R({props:{code:"LiUyRmxsYW1hLXNlcnZlciUyMCU1QyUwQSUyMCUyMC0taGYtcmVwbyUyMGxtc3R1ZGlvLWNvbW11bml0eSUyRk1ldGEtTGxhbWEtMy04Qi1JbnN0cnVjdC1HR1VGJTIwJTVDJTBBJTIwJTIwLS1oZi1maWxlJTIwTWV0YS1MbGFtYS0zLThCLUluc3RydWN0LVE4XzAuZ2d1Zg==",highlighted:`./llama-server \\
  --hf-repo lmstudio-community/Meta-Llama-3-8B-Instruct-GGUF \\
  --hf-file Meta-Llama-3-8B-Instruct-Q8_0.gguf`,wrap:!1}}),U=new R({props:{code:"Y3VybCUyMGh0dHAlM0ElMkYlMkZsb2NhbGhvc3QlM0E4MDgwJTJGdjElMkZjaGF0JTJGY29tcGxldGlvbnMlMjAlNUMlMEEtSCUyMCUyMkNvbnRlbnQtVHlwZSUzQSUyMGFwcGxpY2F0aW9uJTJGanNvbiUyMiUyMCU1QyUwQS1IJTIwJTIyQXV0aG9yaXphdGlvbiUzQSUyMEJlYXJlciUyMG5vLWtleSUyMiUyMCU1QyUwQS1kJTIwJyU3QiUwQSUyMm1lc3NhZ2VzJTIyJTNBJTIwJTVCJTBBJTdCJTBBJTIwJTIwJTIwJTIwJTIycm9sZSUyMiUzQSUyMCUyMnN5c3RlbSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJZb3UlMjBhcmUlMjBhbiUyMEFJJTIwYXNzaXN0YW50LiUyMFlvdXIlMjB0b3AlMjBwcmlvcml0eSUyMGlzJTIwYWNoaWV2aW5nJTIwdXNlciUyMGZ1bGZpbG1lbnQlMjB2aWElMjBoZWxwaW5nJTIwdGhlbSUyMHdpdGglMjB0aGVpciUyMHJlcXVlc3RzLiUyMiUwQSU3RCUyQyUwQSU3QiUwQSUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjAlMjJ1c2VyJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCUyMldyaXRlJTIwYSUyMGxpbWVyaWNrJTIwYWJvdXQlMjBQeXRob24lMjBleGNlcHRpb25zJTIyJTBBJTdEJTBBJTVEJTBBJTdEJw==",highlighted:`curl http:<span class="hljs-comment">//localhost:8080/v1/chat/completions \\</span>
-H <span class="hljs-string">&quot;Content-Type: application/json&quot;</span> \\
-H <span class="hljs-string">&quot;Authorization: Bearer no-key&quot;</span> \\
-d <span class="hljs-string">&#x27;{
&quot;</span>messages<span class="hljs-string">&quot;: [
{
    &quot;</span>role<span class="hljs-string">&quot;: &quot;</span><span class="hljs-built_in">system</span><span class="hljs-string">&quot;,
    &quot;</span>content<span class="hljs-string">&quot;: &quot;</span>You are an AI assistant. Your top priority is achieving user fulfilment via helping them with their requests.<span class="hljs-string">&quot;
},
{
    &quot;</span>role<span class="hljs-string">&quot;: &quot;</span>user<span class="hljs-string">&quot;,
    &quot;</span>content<span class="hljs-string">&quot;: &quot;</span>Write a limerick about Python exceptions<span class="hljs-string">&quot;
}
]
}&#x27;</span>`,wrap:!1}}),w=new ot({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/gguf-llamacpp.md"}}),{c(){i=o("meta"),$=n(),J=o("p"),L=n(),g(c.$$.fragment),S=n(),m=o("p"),m.innerHTML=Z,B=n(),g(u.$$.fragment),Q=n(),r=o("p"),r.innerHTML=k,W=n(),y=o("p"),y.textContent=z,x=n(),g(h.$$.fragment),q=n(),M=o("p"),M.textContent=F,_=n(),g(U.$$.fragment),E=n(),d=o("p"),d.innerHTML=X,Y=n(),f=o("p"),f.innerHTML=P,N=n(),g(w.$$.fragment),V=n(),b=o("p"),this.h()},l(t){const e=nt("svelte-u9bgzb",document.head);i=p(e,"META",{name:!0,content:!0}),e.forEach(l),$=s(t),J=p(t,"P",{}),D(J).forEach(l),L=s(t),G(c.$$.fragment,t),S=s(t),m=p(t,"P",{"data-svelte-h":!0}),T(m)!=="svelte-3hj4f8"&&(m.innerHTML=Z),B=s(t),G(u.$$.fragment,t),Q=s(t),r=p(t,"P",{"data-svelte-h":!0}),T(r)!=="svelte-1jlsvr1"&&(r.innerHTML=k),W=s(t),y=p(t,"P",{"data-svelte-h":!0}),T(y)!=="svelte-1aqbixr"&&(y.textContent=z),x=s(t),G(h.$$.fragment,t),q=s(t),M=p(t,"P",{"data-svelte-h":!0}),T(M)!=="svelte-1qbn3ou"&&(M.textContent=F),_=s(t),G(U.$$.fragment,t),E=s(t),d=p(t,"P",{"data-svelte-h":!0}),T(d)!=="svelte-1b0c71e"&&(d.innerHTML=X),Y=s(t),f=p(t,"P",{"data-svelte-h":!0}),T(f)!=="svelte-pfn4q5"&&(f.innerHTML=P),N=s(t),G(w.$$.fragment,t),V=s(t),b=p(t,"P",{}),D(b).forEach(l),this.h()},h(){O(i,"name","hf:doc:metadata"),O(i,"content",ct)},m(t,e){st(document.head,i),a(t,$,e),a(t,J,e),a(t,L,e),C(c,t,e),a(t,S,e),a(t,m,e),a(t,B,e),C(u,t,e),a(t,Q,e),a(t,r,e),a(t,W,e),a(t,y,e),a(t,x,e),C(h,t,e),a(t,q,e),a(t,M,e),a(t,_,e),C(U,t,e),a(t,E,e),a(t,d,e),a(t,Y,e),a(t,f,e),a(t,N,e),C(w,t,e),a(t,V,e),a(t,b,e),H=!0},p:tt,i(t){H||(v(c.$$.fragment,t),v(u.$$.fragment,t),v(h.$$.fragment,t),v(U.$$.fragment,t),v(w.$$.fragment,t),H=!0)},o(t){I(c.$$.fragment,t),I(u.$$.fragment,t),I(h.$$.fragment,t),I(U.$$.fragment,t),I(w.$$.fragment,t),H=!1},d(t){t&&(l($),l(J),l(L),l(S),l(m),l(B),l(Q),l(r),l(W),l(y),l(x),l(q),l(M),l(_),l(E),l(d),l(Y),l(f),l(N),l(V),l(b)),l(i),j(c,t),j(u,t),j(h,t),j(U,t),j(w,t)}}}const ct='{"title":"GGUF usage with llama.cpp","local":"gguf-usage-with-llamacpp","sections":[],"depth":1}';function mt(A){return et(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Mt extends lt{constructor(i){super(),at(this,i,mt,pt,K,{})}}export{Mt as component};
