import{s as Xt,n as Ht,o as Rt}from"../chunks/scheduler.25b97de1.js";import{S as Wt,i as Qt,g as a,s,r,A as Yt,h as o,f as l,c as i,j as Zt,u as M,x as p,k as Lt,y as Ft,a as n,v as w,d as m,t as y,w as h}from"../chunks/index.d9030fc9.js";import{C as re}from"../chunks/CodeBlock.e6cd0d95.js";import{H as Me,E as zt}from"../chunks/EditOnGithub.91d95064.js";function Dt(it){let c,we,oe,me,d,ye,u,at='The hardware you use to run model training and inference can have a big effect on performance. For a deep dive into GPUs make sure to check out Tim Dettmer’s excellent <a href="https://timdettmers.com/2020/09/07/which-gpu-for-deep-learning/" rel="nofollow">blog post</a>.',he,U,ot="Let’s have a look at some practical advice for GPU setups.",ce,C,de,f,pt="When you train bigger models you have essentially three options:",ue,T,rt='<li>bigger GPUs</li> <li>more GPUs</li> <li>more CPU and NVMe (offloaded to by <a href="main_classes/deepspeed#nvme-support">DeepSpeed-Infinity</a>)</li>',Ue,J,Mt="Let’s start at the case where you have a single GPU.",Ce,I,fe,b,wt="If you bought an expensive high end GPU make sure you give it the correct power and sufficient cooling.",Te,v,mt="<strong>Power</strong>:",Je,j,yt="Some high end consumer GPU cards have 2 and sometimes 3 PCI-E 8-Pin power sockets. Make sure you have as many independent 12V PCI-E 8-Pin cables plugged into the card as there are sockets. Do not use the 2 splits at one end of the same cable (also known as pigtail cable). That is if you have 2 sockets on the GPU, you want 2 PCI-E 8-Pin cables going from your PSU to the card and not one that has 2 PCI-E 8-Pin connectors at the end! You won’t get the full performance out of your card otherwise.",Ie,x,ht="Each PCI-E 8-Pin power cable needs to be plugged into a 12V rail on the PSU side and can supply up to 150W of power.",be,P,ct="Some other cards may use a PCI-E 12-Pin connectors, and these can deliver up to 500-600W of power.",ve,k,dt="Low end cards may use 6-Pin connectors, which supply up to 75W of power.",je,g,ut="Additionally you want the high-end PSU that has stable voltage. Some lower quality ones may not give the card the stable voltage it needs to function at its peak.",xe,B,Ut="And of course the PSU needs to have enough unused Watts to power the card.",Pe,_,Ct="<strong>Cooling</strong>:",ke,N,ft="When a GPU gets overheated it will start throttling down and will not deliver full performance and it can even shutdown if it gets too hot.",ge,G,Tt="It’s hard to tell the exact best temperature to strive for when a GPU is heavily loaded, but probably anything under +80C is good, but lower is better - perhaps 70-75C is an excellent range to be in. The throttling down is likely to start at around 84-90C. But other than throttling performance a prolonged very high temperature is likely to reduce the lifespan of a GPU.",Be,V,Jt="Next let’s have a look at one of the most important aspects when having multiple GPUs: connectivity.",_e,$,Ne,A,It="If you use multiple GPUs the way cards are inter-connected can have a huge impact on the total training time. If the GPUs are on the same physical node, you can run:",Ge,E,Ve,S,bt="and it will tell you how the GPUs are inter-connected. On a machine with dual-GPU and which are connected with NVLink, you will most likely see something like:",$e,Z,Ae,L,vt="on a different machine w/o NVLink we may see:",Ee,X,Se,H,jt="The report includes this legend:",Ze,R,Le,W,xt="So the first report <code>NV2</code> tells us the GPUs are interconnected with 2 NVLinks, and the second report <code>PHB</code> we have a typical consumer-level PCIe+Bridge setup.",Xe,Q,Pt="Check what type of connectivity you have on your setup. Some of these will make the communication between cards faster (e.g. NVLink), others slower (e.g. PHB).",He,Y,kt="Depending on the type of scalability solution used, the connectivity speed could have a major or a minor impact. If the GPUs need to sync rarely, as in DDP, the impact of a slower connection will be less significant. If the GPUs need to send messages to each other often, as in ZeRO-DP, then faster connectivity becomes super important to achieve faster training.",Re,F,We,z,gt='<a href="https://en.wikipedia.org/wiki/NVLink" rel="nofollow">NVLink</a> is a wire-based serial multi-lane near-range communications link developed by Nvidia.',Qe,D,Bt='Each new generation provides a faster bandwidth, e.g. here is a quote from <a href="https://www.nvidia.com/content/dam/en-zz/Solutions/geforce/ampere/pdf/NVIDIA-ampere-GA102-GPU-Architecture-Whitepaper-V1.pdf" rel="nofollow">Nvidia Ampere GA102 GPU Architecture</a>:',Ye,O,_t=`<p>Third-Generation NVLink®
GA102 GPUs utilize NVIDIA’s third-generation NVLink interface, which includes four x4 links,
with each link providing 14.0625 GB/sec bandwidth in each direction between two GPUs. Four
links provide 56.25 GB/sec bandwidth in each direction, and 112.5 GB/sec total bandwidth
between two GPUs. Two RTX 3090 GPUs can be connected together for SLI using NVLink.
(Note that 3-Way and 4-Way SLI configurations are not supported.)</p>`,Fe,q,Nt="So the higher <code>X</code> you get in the report of <code>NVX</code> in the output of <code>nvidia-smi topo -m</code> the better. The generation will depend on your GPU architecture.",ze,K,Gt="Let’s compare the execution of a openai-community/gpt2 language model training over a small sample of wikitext.",De,ee,Vt="The results are:",Oe,te,$t='<thead><tr><th>NVlink</th> <th align="right">Time</th></tr></thead> <tbody><tr><td>Y</td> <td align="right">101s</td></tr> <tr><td>N</td> <td align="right">131s</td></tr></tbody>',qe,le,At="You can see that NVLink completes the training ~23% faster. In the second benchmark we use <code>NCCL_P2P_DISABLE=1</code> to tell the GPUs not to use NVLink.",Ke,ne,Et="Here is the full benchmark code and outputs:",et,se,tt,ie,St=`Hardware: 2x TITAN RTX 24GB each + NVlink with 2 NVLinks (<code>NV2</code> in <code>nvidia-smi topo -m</code>)
Software: <code>pytorch-1.8-to-be</code> + <code>cuda-11.0</code> / <code>transformers==4.3.0.dev0</code>`,lt,ae,nt,pe,st;return d=new Me({props:{title:"Custom hardware for training",local:"custom-hardware-for-training",headingTag:"h1"}}),C=new Me({props:{title:"GPU",local:"gpu",headingTag:"h2"}}),I=new Me({props:{title:"Power and Cooling",local:"power-and-cooling",headingTag:"h3"}}),$=new Me({props:{title:"Multi-GPU Connectivity",local:"multi-gpu-connectivity",headingTag:"h3"}}),E=new re({props:{code:"bnZpZGlhLXNtaSUyMHRvcG8lMjAtbQ==",highlighted:"nvidia-smi topo -m",wrap:!1}}),Z=new re({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwR1BVMCUyMCUyMCUyMCUyMEdQVTElMjAlMjAlMjAlMjBDUFUlMjBBZmZpbml0eSUyMCUyMCUyMCUyME5VTUElMjBBZmZpbml0eSUwQUdQVTAlMjAlMjAlMjAlMjAlMjBYJTIwJTIwJTIwJTIwJTIwJTIwTlYyJTIwJTIwJTIwJTIwJTIwMC0yMyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME4lMkZBJTBBR1BVMSUyMCUyMCUyMCUyME5WMiUyMCUyMCUyMCUyMCUyMCUyMFglMjAlMjAlMjAlMjAlMjAlMjAwLTIzJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTiUyRkE=",highlighted:`        <span class="hljs-attribute">GPU0</span>    GPU1    CPU Affinity    NUMA Affinity
<span class="hljs-attribute">GPU0</span>     X      NV2     <span class="hljs-number">0</span>-<span class="hljs-number">23</span>            N/A
<span class="hljs-attribute">GPU1</span>    NV2      X      <span class="hljs-number">0</span>-<span class="hljs-number">23</span>            N/A`,wrap:!1}}),X=new re({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwR1BVMCUyMCUyMCUyMCUyMEdQVTElMjAlMjAlMjAlMjBDUFUlMjBBZmZpbml0eSUyMCUyMCUyMCUyME5VTUElMjBBZmZpbml0eSUwQUdQVTAlMjAlMjAlMjAlMjAlMjBYJTIwJTIwJTIwJTIwJTIwJTIwUEhCJTIwJTIwJTIwJTIwJTIwMC0xMSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME4lMkZBJTBBR1BVMSUyMCUyMCUyMCUyMFBIQiUyMCUyMCUyMCUyMCUyMCUyMFglMjAlMjAlMjAlMjAlMjAlMjAwLTExJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTiUyRkE=",highlighted:`        <span class="hljs-attribute">GPU0</span>    GPU1    CPU Affinity    NUMA Affinity
<span class="hljs-attribute">GPU0</span>     X      PHB     <span class="hljs-number">0</span>-<span class="hljs-number">11</span>            N/A
<span class="hljs-attribute">GPU1</span>    PHB      X      <span class="hljs-number">0</span>-<span class="hljs-number">11</span>            N/A`,wrap:!1}}),R=new re({props:{code:"JTIwJTIwWCUyMCUyMCUyMCUyMCUzRCUyMFNlbGYlMEElMjAlMjBTWVMlMjAlMjAlM0QlMjBDb25uZWN0aW9uJTIwdHJhdmVyc2luZyUyMFBDSWUlMjBhcyUyMHdlbGwlMjBhcyUyMHRoZSUyMFNNUCUyMGludGVyY29ubmVjdCUyMGJldHdlZW4lMjBOVU1BJTIwbm9kZXMlMjAoZS5nLiUyQyUyMFFQSSUyRlVQSSklMEElMjAlMjBOT0RFJTIwJTNEJTIwQ29ubmVjdGlvbiUyMHRyYXZlcnNpbmclMjBQQ0llJTIwYXMlMjB3ZWxsJTIwYXMlMjB0aGUlMjBpbnRlcmNvbm5lY3QlMjBiZXR3ZWVuJTIwUENJZSUyMEhvc3QlMjBCcmlkZ2VzJTIwd2l0aGluJTIwYSUyME5VTUElMjBub2RlJTBBJTIwJTIwUEhCJTIwJTIwJTNEJTIwQ29ubmVjdGlvbiUyMHRyYXZlcnNpbmclMjBQQ0llJTIwYXMlMjB3ZWxsJTIwYXMlMjBhJTIwUENJZSUyMEhvc3QlMjBCcmlkZ2UlMjAodHlwaWNhbGx5JTIwdGhlJTIwQ1BVKSUwQSUyMCUyMFBYQiUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwbXVsdGlwbGUlMjBQQ0llJTIwYnJpZGdlcyUyMCh3aXRob3V0JTIwdHJhdmVyc2luZyUyMHRoZSUyMFBDSWUlMjBIb3N0JTIwQnJpZGdlKSUwQSUyMCUyMFBJWCUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwYXQlMjBtb3N0JTIwYSUyMHNpbmdsZSUyMFBDSWUlMjBicmlkZ2UlMEElMjAlMjBOViUyMyUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwYSUyMGJvbmRlZCUyMHNldCUyMG9mJTIwJTIzJTIwTlZMaW5rcw==",highlighted:`  X    = Self
  SYS  = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">the</span> SMP interconnect between NUMA nodes (e.g., QPI/UPI)
  NODE = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">the</span> interconnect between PCIe Host Bridges <span class="hljs-keyword">within</span> <span class="hljs-keyword">a</span> NUMA node
  PHB  = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">a</span> PCIe Host Bridge (typically <span class="hljs-keyword">the</span> CPU)
  PXB  = Connection traversing multiple PCIe bridges (<span class="hljs-keyword">without</span> traversing <span class="hljs-keyword">the</span> PCIe Host Bridge)
  PIX  = Connection traversing <span class="hljs-keyword">at</span> most <span class="hljs-keyword">a</span> single PCIe bridge
  NV<span class="hljs-comment">#  = Connection traversing a bonded set of # NVLinks</span>`,wrap:!1}}),F=new Me({props:{title:"NVlink",local:"nvlink",headingTag:"h4"}}),se=new re({props:{code:"JTIzJTIwRERQJTIwdyUyRiUyME5WTGluayUwQSUwQXJtJTIwLXIlMjAlMkZ0bXAlMkZ0ZXN0LWNsbSUzQiUyMENVREFfVklTSUJMRV9ERVZJQ0VTJTNEMCUyQzElMjB0b3JjaHJ1biUyMCU1QyUwQS0tbnByb2NfcGVyX25vZGUlMjAyJTIwZXhhbXBsZXMlMkZweXRvcmNoJTJGbGFuZ3VhZ2UtbW9kZWxpbmclMkZydW5fY2xtLnB5JTIwLS1tb2RlbF9uYW1lX29yX3BhdGglMjBvcGVuYWktY29tbXVuaXR5JTJGZ3B0MiUyMCU1QyUwQS0tZGF0YXNldF9uYW1lJTIwd2lraXRleHQlMjAtLWRhdGFzZXRfY29uZmlnX25hbWUlMjB3aWtpdGV4dC0yLXJhdy12MSUyMC0tZG9fdHJhaW4lMjAlNUMlMEEtLW91dHB1dF9kaXIlMjAlMkZ0bXAlMkZ0ZXN0LWNsbSUyMC0tcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIwNCUyMC0tbWF4X3N0ZXBzJTIwMjAwJTBBJTBBJTdCJ3RyYWluX3J1bnRpbWUnJTNBJTIwMTAxLjkwMDMlMkMlMjAndHJhaW5fc2FtcGxlc19wZXJfc2Vjb25kJyUzQSUyMDEuOTYzJTJDJTIwJ2Vwb2NoJyUzQSUyMDAuNjklN0QlMEElMEElMjMlMjBERFAlMjB3JTJGbyUyME5WTGluayUwQSUwQXJtJTIwLXIlMjAlMkZ0bXAlMkZ0ZXN0LWNsbSUzQiUyMENVREFfVklTSUJMRV9ERVZJQ0VTJTNEMCUyQzElMjBOQ0NMX1AyUF9ESVNBQkxFJTNEMSUyMHRvcmNocnVuJTIwJTVDJTBBLS1ucHJvY19wZXJfbm9kZSUyMDIlMjBleGFtcGxlcyUyRnB5dG9yY2glMkZsYW5ndWFnZS1tb2RlbGluZyUyRnJ1bl9jbG0ucHklMjAtLW1vZGVsX25hbWVfb3JfcGF0aCUyMG9wZW5haS1jb21tdW5pdHklMkZncHQyJTIwJTVDJTBBLS1kYXRhc2V0X25hbWUlMjB3aWtpdGV4dCUyMC0tZGF0YXNldF9jb25maWdfbmFtZSUyMHdpa2l0ZXh0LTItcmF3LXYxJTIwLS1kb190cmFpbiUwQS0tb3V0cHV0X2RpciUyMCUyRnRtcCUyRnRlc3QtY2xtJTIwLS1wZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlMjA0JTIwLS1tYXhfc3RlcHMlMjAyMDAlMEElMEElN0IndHJhaW5fcnVudGltZSclM0ElMjAxMzEuNDM2NyUyQyUyMCd0cmFpbl9zYW1wbGVzX3Blcl9zZWNvbmQnJTNBJTIwMS41MjIlMkMlMjAnZXBvY2gnJTNBJTIwMC42OSU3RA==",highlighted:`<span class="hljs-comment"># DDP w/ NVLink</span>

<span class="hljs-built_in">rm</span> -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 torchrun \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path openai-community/gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train \\
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: 101.9003, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: 1.963, <span class="hljs-string">&#x27;epoch&#x27;</span>: 0.69}

<span class="hljs-comment"># DDP w/o NVLink</span>

<span class="hljs-built_in">rm</span> -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 NCCL_P2P_DISABLE=1 torchrun \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path openai-community/gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: 131.4367, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: 1.522, <span class="hljs-string">&#x27;epoch&#x27;</span>: 0.69}`,wrap:!1}}),ae=new zt({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/perf_hardware.md"}}),{c(){c=a("meta"),we=s(),oe=a("p"),me=s(),r(d.$$.fragment),ye=s(),u=a("p"),u.innerHTML=at,he=s(),U=a("p"),U.textContent=ot,ce=s(),r(C.$$.fragment),de=s(),f=a("p"),f.textContent=pt,ue=s(),T=a("ul"),T.innerHTML=rt,Ue=s(),J=a("p"),J.textContent=Mt,Ce=s(),r(I.$$.fragment),fe=s(),b=a("p"),b.textContent=wt,Te=s(),v=a("p"),v.innerHTML=mt,Je=s(),j=a("p"),j.textContent=yt,Ie=s(),x=a("p"),x.textContent=ht,be=s(),P=a("p"),P.textContent=ct,ve=s(),k=a("p"),k.textContent=dt,je=s(),g=a("p"),g.textContent=ut,xe=s(),B=a("p"),B.textContent=Ut,Pe=s(),_=a("p"),_.innerHTML=Ct,ke=s(),N=a("p"),N.textContent=ft,ge=s(),G=a("p"),G.textContent=Tt,Be=s(),V=a("p"),V.textContent=Jt,_e=s(),r($.$$.fragment),Ne=s(),A=a("p"),A.textContent=It,Ge=s(),r(E.$$.fragment),Ve=s(),S=a("p"),S.textContent=bt,$e=s(),r(Z.$$.fragment),Ae=s(),L=a("p"),L.textContent=vt,Ee=s(),r(X.$$.fragment),Se=s(),H=a("p"),H.textContent=jt,Ze=s(),r(R.$$.fragment),Le=s(),W=a("p"),W.innerHTML=xt,Xe=s(),Q=a("p"),Q.textContent=Pt,He=s(),Y=a("p"),Y.textContent=kt,Re=s(),r(F.$$.fragment),We=s(),z=a("p"),z.innerHTML=gt,Qe=s(),D=a("p"),D.innerHTML=Bt,Ye=s(),O=a("blockquote"),O.innerHTML=_t,Fe=s(),q=a("p"),q.innerHTML=Nt,ze=s(),K=a("p"),K.textContent=Gt,De=s(),ee=a("p"),ee.textContent=Vt,Oe=s(),te=a("table"),te.innerHTML=$t,qe=s(),le=a("p"),le.innerHTML=At,Ke=s(),ne=a("p"),ne.textContent=Et,et=s(),r(se.$$.fragment),tt=s(),ie=a("p"),ie.innerHTML=St,lt=s(),r(ae.$$.fragment),nt=s(),pe=a("p"),this.h()},l(e){const t=Yt("svelte-u9bgzb",document.head);c=o(t,"META",{name:!0,content:!0}),t.forEach(l),we=i(e),oe=o(e,"P",{}),Zt(oe).forEach(l),me=i(e),M(d.$$.fragment,e),ye=i(e),u=o(e,"P",{"data-svelte-h":!0}),p(u)!=="svelte-iifysf"&&(u.innerHTML=at),he=i(e),U=o(e,"P",{"data-svelte-h":!0}),p(U)!=="svelte-1dshoiu"&&(U.textContent=ot),ce=i(e),M(C.$$.fragment,e),de=i(e),f=o(e,"P",{"data-svelte-h":!0}),p(f)!=="svelte-eleqzp"&&(f.textContent=pt),ue=i(e),T=o(e,"UL",{"data-svelte-h":!0}),p(T)!=="svelte-pz3vuj"&&(T.innerHTML=rt),Ue=i(e),J=o(e,"P",{"data-svelte-h":!0}),p(J)!=="svelte-z1tcso"&&(J.textContent=Mt),Ce=i(e),M(I.$$.fragment,e),fe=i(e),b=o(e,"P",{"data-svelte-h":!0}),p(b)!=="svelte-ppjikm"&&(b.textContent=wt),Te=i(e),v=o(e,"P",{"data-svelte-h":!0}),p(v)!=="svelte-11nr6dq"&&(v.innerHTML=mt),Je=i(e),j=o(e,"P",{"data-svelte-h":!0}),p(j)!=="svelte-11nhv2k"&&(j.textContent=yt),Ie=i(e),x=o(e,"P",{"data-svelte-h":!0}),p(x)!=="svelte-150xmew"&&(x.textContent=ht),be=i(e),P=o(e,"P",{"data-svelte-h":!0}),p(P)!=="svelte-12gfejp"&&(P.textContent=ct),ve=i(e),k=o(e,"P",{"data-svelte-h":!0}),p(k)!=="svelte-uh6lop"&&(k.textContent=dt),je=i(e),g=o(e,"P",{"data-svelte-h":!0}),p(g)!=="svelte-s9j0io"&&(g.textContent=ut),xe=i(e),B=o(e,"P",{"data-svelte-h":!0}),p(B)!=="svelte-8g3p2u"&&(B.textContent=Ut),Pe=i(e),_=o(e,"P",{"data-svelte-h":!0}),p(_)!=="svelte-1gvtzcc"&&(_.innerHTML=Ct),ke=i(e),N=o(e,"P",{"data-svelte-h":!0}),p(N)!=="svelte-hba1wh"&&(N.textContent=ft),ge=i(e),G=o(e,"P",{"data-svelte-h":!0}),p(G)!=="svelte-1ho6s7e"&&(G.textContent=Tt),Be=i(e),V=o(e,"P",{"data-svelte-h":!0}),p(V)!=="svelte-1c43r32"&&(V.textContent=Jt),_e=i(e),M($.$$.fragment,e),Ne=i(e),A=o(e,"P",{"data-svelte-h":!0}),p(A)!=="svelte-1b7hx6s"&&(A.textContent=It),Ge=i(e),M(E.$$.fragment,e),Ve=i(e),S=o(e,"P",{"data-svelte-h":!0}),p(S)!=="svelte-1m7rqth"&&(S.textContent=bt),$e=i(e),M(Z.$$.fragment,e),Ae=i(e),L=o(e,"P",{"data-svelte-h":!0}),p(L)!=="svelte-hggxkn"&&(L.textContent=vt),Ee=i(e),M(X.$$.fragment,e),Se=i(e),H=o(e,"P",{"data-svelte-h":!0}),p(H)!=="svelte-869n3z"&&(H.textContent=jt),Ze=i(e),M(R.$$.fragment,e),Le=i(e),W=o(e,"P",{"data-svelte-h":!0}),p(W)!=="svelte-xobw9p"&&(W.innerHTML=xt),Xe=i(e),Q=o(e,"P",{"data-svelte-h":!0}),p(Q)!=="svelte-abx4jy"&&(Q.textContent=Pt),He=i(e),Y=o(e,"P",{"data-svelte-h":!0}),p(Y)!=="svelte-1hcsw7p"&&(Y.textContent=kt),Re=i(e),M(F.$$.fragment,e),We=i(e),z=o(e,"P",{"data-svelte-h":!0}),p(z)!=="svelte-1gqi5fx"&&(z.innerHTML=gt),Qe=i(e),D=o(e,"P",{"data-svelte-h":!0}),p(D)!=="svelte-gh3n8e"&&(D.innerHTML=Bt),Ye=i(e),O=o(e,"BLOCKQUOTE",{"data-svelte-h":!0}),p(O)!=="svelte-14khom1"&&(O.innerHTML=_t),Fe=i(e),q=o(e,"P",{"data-svelte-h":!0}),p(q)!=="svelte-1naftur"&&(q.innerHTML=Nt),ze=i(e),K=o(e,"P",{"data-svelte-h":!0}),p(K)!=="svelte-137n5ps"&&(K.textContent=Gt),De=i(e),ee=o(e,"P",{"data-svelte-h":!0}),p(ee)!=="svelte-j6eczh"&&(ee.textContent=Vt),Oe=i(e),te=o(e,"TABLE",{"data-svelte-h":!0}),p(te)!=="svelte-1hvmx1i"&&(te.innerHTML=$t),qe=i(e),le=o(e,"P",{"data-svelte-h":!0}),p(le)!=="svelte-192o6bz"&&(le.innerHTML=At),Ke=i(e),ne=o(e,"P",{"data-svelte-h":!0}),p(ne)!=="svelte-16jny0z"&&(ne.textContent=Et),et=i(e),M(se.$$.fragment,e),tt=i(e),ie=o(e,"P",{"data-svelte-h":!0}),p(ie)!=="svelte-pittpg"&&(ie.innerHTML=St),lt=i(e),M(ae.$$.fragment,e),nt=i(e),pe=o(e,"P",{}),Zt(pe).forEach(l),this.h()},h(){Lt(c,"name","hf:doc:metadata"),Lt(c,"content",Ot)},m(e,t){Ft(document.head,c),n(e,we,t),n(e,oe,t),n(e,me,t),w(d,e,t),n(e,ye,t),n(e,u,t),n(e,he,t),n(e,U,t),n(e,ce,t),w(C,e,t),n(e,de,t),n(e,f,t),n(e,ue,t),n(e,T,t),n(e,Ue,t),n(e,J,t),n(e,Ce,t),w(I,e,t),n(e,fe,t),n(e,b,t),n(e,Te,t),n(e,v,t),n(e,Je,t),n(e,j,t),n(e,Ie,t),n(e,x,t),n(e,be,t),n(e,P,t),n(e,ve,t),n(e,k,t),n(e,je,t),n(e,g,t),n(e,xe,t),n(e,B,t),n(e,Pe,t),n(e,_,t),n(e,ke,t),n(e,N,t),n(e,ge,t),n(e,G,t),n(e,Be,t),n(e,V,t),n(e,_e,t),w($,e,t),n(e,Ne,t),n(e,A,t),n(e,Ge,t),w(E,e,t),n(e,Ve,t),n(e,S,t),n(e,$e,t),w(Z,e,t),n(e,Ae,t),n(e,L,t),n(e,Ee,t),w(X,e,t),n(e,Se,t),n(e,H,t),n(e,Ze,t),w(R,e,t),n(e,Le,t),n(e,W,t),n(e,Xe,t),n(e,Q,t),n(e,He,t),n(e,Y,t),n(e,Re,t),w(F,e,t),n(e,We,t),n(e,z,t),n(e,Qe,t),n(e,D,t),n(e,Ye,t),n(e,O,t),n(e,Fe,t),n(e,q,t),n(e,ze,t),n(e,K,t),n(e,De,t),n(e,ee,t),n(e,Oe,t),n(e,te,t),n(e,qe,t),n(e,le,t),n(e,Ke,t),n(e,ne,t),n(e,et,t),w(se,e,t),n(e,tt,t),n(e,ie,t),n(e,lt,t),w(ae,e,t),n(e,nt,t),n(e,pe,t),st=!0},p:Ht,i(e){st||(m(d.$$.fragment,e),m(C.$$.fragment,e),m(I.$$.fragment,e),m($.$$.fragment,e),m(E.$$.fragment,e),m(Z.$$.fragment,e),m(X.$$.fragment,e),m(R.$$.fragment,e),m(F.$$.fragment,e),m(se.$$.fragment,e),m(ae.$$.fragment,e),st=!0)},o(e){y(d.$$.fragment,e),y(C.$$.fragment,e),y(I.$$.fragment,e),y($.$$.fragment,e),y(E.$$.fragment,e),y(Z.$$.fragment,e),y(X.$$.fragment,e),y(R.$$.fragment,e),y(F.$$.fragment,e),y(se.$$.fragment,e),y(ae.$$.fragment,e),st=!1},d(e){e&&(l(we),l(oe),l(me),l(ye),l(u),l(he),l(U),l(ce),l(de),l(f),l(ue),l(T),l(Ue),l(J),l(Ce),l(fe),l(b),l(Te),l(v),l(Je),l(j),l(Ie),l(x),l(be),l(P),l(ve),l(k),l(je),l(g),l(xe),l(B),l(Pe),l(_),l(ke),l(N),l(ge),l(G),l(Be),l(V),l(_e),l(Ne),l(A),l(Ge),l(Ve),l(S),l($e),l(Ae),l(L),l(Ee),l(Se),l(H),l(Ze),l(Le),l(W),l(Xe),l(Q),l(He),l(Y),l(Re),l(We),l(z),l(Qe),l(D),l(Ye),l(O),l(Fe),l(q),l(ze),l(K),l(De),l(ee),l(Oe),l(te),l(qe),l(le),l(Ke),l(ne),l(et),l(tt),l(ie),l(lt),l(nt),l(pe)),l(c),h(d,e),h(C,e),h(I,e),h($,e),h(E,e),h(Z,e),h(X,e),h(R,e),h(F,e),h(se,e),h(ae,e)}}}const Ot='{"title":"Custom hardware for training","local":"custom-hardware-for-training","sections":[{"title":"GPU","local":"gpu","sections":[{"title":"Power and Cooling","local":"power-and-cooling","sections":[],"depth":3},{"title":"Multi-GPU Connectivity","local":"multi-gpu-connectivity","sections":[{"title":"NVlink","local":"nvlink","sections":[],"depth":4}],"depth":3}],"depth":2}],"depth":1}';function qt(it){return Rt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class nl extends Wt{constructor(c){super(),Qt(this,c,qt,Dt,Xt,{})}}export{nl as component};
