import{s as jt,o as wt,n as A}from"../chunks/scheduler.9bc65507.js";import{S as yt,i as Ut,g as c,s as M,r as m,A as dt,h as i,f as e,c as p,j as ot,u as j,x as T,k as Jt,y as ut,a as t,v as w,d as y,t as U,w as d}from"../chunks/index.707bf1b6.js";import{T as Kl}from"../chunks/Tip.c2ecdbf4.js";import{C as g}from"../chunks/CodeBlock.54a9f38d.js";import{H as Z,E as ft}from"../chunks/EditOnGithub.922df6ba.js";import{H as mt,a as ql}from"../chunks/HfOption.6d864328.js";function ht(h){let a,r='Transformers는 <code>Trainer</code> 클래스 외에도 번역이나 요약과 같은 시퀀스-투-시퀀스 작업을 위한 <code>Seq2SeqTrainer</code> 클래스도 제공합니다. 또한 <a href="https://hf.co/docs/trl" rel="nofollow">TRL</a> 라이브러리에는 <code>Trainer</code> 클래스를 감싸고 Llama-2 및 Mistral과 같은 언어 모델을 자동 회귀 기법으로 훈련하는 데 최적화된 <code>SFTTrainer</code> 클래스 입니다. <code>SFTTrainer</code>는 시퀀스 패킹, LoRA, 양자화 및 DeepSpeed와 같은 기능을 지원하여 크기 상관없이 모델 효율적으로 확장할 수 있습니다.',n,u,o,f,b='이들 다른 <code>Trainer</code> 유형 클래스에 대해 더 알고 싶다면 <a href="./main_classes/trainer">API 참조</a>를 확인하여 언제 어떤 클래스가 적합할지 얼마든지 확인하세요. 일반적으로 <code>Trainer</code>는 가장 다재다능한 옵션으로, 다양한 작업에 적합합니다. <code>Seq2SeqTrainer</code>는 시퀀스-투-시퀀스 작업을 위해 설계되었고, <code>SFTTrainer</code>는 언어 모델 훈련을 위해 설계되었습니다.';return{c(){a=c("p"),a.innerHTML=r,n=M(),u=c("br"),o=M(),f=c("p"),f.innerHTML=b},l(_){a=i(_,"P",{"data-svelte-h":!0}),T(a)!=="svelte-26hx6t"&&(a.innerHTML=r),n=p(_),u=i(_,"BR",{}),o=p(_),f=i(_,"P",{"data-svelte-h":!0}),T(f)!=="svelte-7uuu5x"&&(f.innerHTML=b)},m(_,J){t(_,a,J),t(_,n,J),t(_,u,J),t(_,o,J),t(_,f,J)},p:A,d(_){_&&(e(a),e(n),e(u),e(o),e(f))}}}function bt(h){let a,r='로깅 API에 대한 자세한 내용은 <a href="./main_classes/logging">로깅</a> API 레퍼런스를 확인하세요.';return{c(){a=c("p"),a.innerHTML=r},l(n){a=i(n,"P",{"data-svelte-h":!0}),T(a)!=="svelte-1djg64a"&&(a.innerHTML=r)},m(n,u){t(n,a,u)},p:A,d(n){n&&e(a)}}}function It(h){let a,r="<code>Trainer</code>는 <code>Trainer.__init__()</code> 메소드에서 각 노드에 대해 로그 레벨을 별도로 설정하므로, 다른 Transformers 기능을 사용할 경우 <code>Trainer</code> 객체를 생성하기 전에 이를 미리 설정하는 것이 좋습니다.";return{c(){a=c("p"),a.innerHTML=r},l(n){a=i(n,"P",{"data-svelte-h":!0}),T(a)!=="svelte-rdkluh"&&(a.innerHTML=r)},m(n,u){t(n,a,u)},p:A,d(n){n&&e(a)}}}function _t(h){let a,r;return a=new g({props:{code:"bXlfYXBwLnB5JTIwLi4uJTIwLS1sb2dfbGV2ZWwlMjB3YXJuaW5nJTIwLS1sb2dfbGV2ZWxfcmVwbGljYSUyMGVycm9y",highlighted:"my_app.py ... --log_level warning --log_level_replica error",wrap:!1}}),{c(){m(a.$$.fragment)},l(n){j(a.$$.fragment,n)},m(n,u){w(a,n,u),r=!0},p:A,i(n){r||(y(a.$$.fragment,n),r=!0)},o(n){U(a.$$.fragment,n),r=!1},d(n){d(a,n)}}}function gt(h){let a,r="멀티 노드 환경에서는 <code>log_on_each_node 0</code> 매개변수를 추가합니다.",n,u,o;return u=new g({props:{code:"bXlfYXBwLnB5JTIwLi4uJTIwLS1sb2dfbGV2ZWwlMjB3YXJuaW5nJTIwLS1sb2dfbGV2ZWxfcmVwbGljYSUyMGVycm9yJTIwLS1sb2dfb25fZWFjaF9ub2RlJTIwMCUwQSUwQSUyMyUyMCVFQyU5OCVBNCVFQiVBNSU5OCVFQiVBNyU4QyUyMCVFQiVCMyVCNCVFQSVCMyVBMCVFRCU5NSU5OCVFQiU4RiU4NCVFQiVBMSU5RCUyMCVFQyU4NCVBNCVFQyVBMCU5NSUwQW15X2FwcC5weSUyMC4uLiUyMC0tbG9nX2xldmVsJTIwZXJyb3IlMjAtLWxvZ19sZXZlbF9yZXBsaWNhJTIwZXJyb3IlMjAtLWxvZ19vbl9lYWNoX25vZGUlMjAw",highlighted:`my_app.py ... --log_level warning --log_level_replica error --log_on_each_node 0

<span class="hljs-comment"># 오류만 보고하도록 설정</span>
my_app.py ... --log_level error --log_level_replica error --log_on_each_node 0`,wrap:!1}}),{c(){a=c("p"),a.innerHTML=r,n=M(),m(u.$$.fragment)},l(f){a=i(f,"P",{"data-svelte-h":!0}),T(a)!=="svelte-1tjciiw"&&(a.innerHTML=r),n=p(f),j(u.$$.fragment,f)},m(f,b){t(f,a,b),t(f,n,b),w(u,f,b),o=!0},p:A,i(f){o||(y(u.$$.fragment,f),o=!0)},o(f){U(u.$$.fragment,f),o=!1},d(f){f&&(e(a),e(n)),d(u,f)}}}function At(h){let a,r,n,u;return a=new ql({props:{id:"logging",option:"single node",$$slots:{default:[_t]},$$scope:{ctx:h}}}),n=new ql({props:{id:"logging",option:"multi-node",$$slots:{default:[gt]},$$scope:{ctx:h}}}),{c(){m(a.$$.fragment),r=M(),m(n.$$.fragment)},l(o){j(a.$$.fragment,o),r=p(o),j(n.$$.fragment,o)},m(o,f){w(a,o,f),t(o,r,f),w(n,o,f),u=!0},p(o,f){const b={};f&2&&(b.$$scope={dirty:f,ctx:o}),a.$set(b);const _={};f&2&&(_.$$scope={dirty:f,ctx:o}),n.$set(_)},i(o){u||(y(a.$$.fragment,o),y(n.$$.fragment,o),u=!0)},o(o){U(a.$$.fragment,o),U(n.$$.fragment,o),u=!1},d(o){o&&e(r),d(a,o),d(n,o)}}}function Ct(h){let a,r="저자에 따르면, <code>grad_norm</code> 없이 <code>AdaLomo</code>를 사용하는 것이 더 나은 성능과 높은 처리량을 제공한다고 합니다.";return{c(){a=c("p"),a.innerHTML=r},l(n){a=i(n,"P",{"data-svelte-h":!0}),T(a)!=="svelte-733wbz"&&(a.innerHTML=r)},m(n,u){t(n,a,u)},p:A,d(n){n&&e(a)}}}function Zt(h){let a,r='FSDP 샤딩 전략, CPU 오프로드 및 <code>Trainer</code>와 함께 사용할 수 있는 더 많은 기능을 알아보려면 <a href="fsdp">Fully Sharded Data Parallel</a> 가이드를 확인하세요.';return{c(){a=c("p"),a.innerHTML=r},l(n){a=i(n,"P",{"data-svelte-h":!0}),T(a)!=="svelte-u7bkxx"&&(a.innerHTML=r)},m(n,u){t(n,a,u)},p:A,d(n){n&&e(a)}}}function Vt(h){let a,r;return a=new g({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMEFkaXN0cmlidXRlZF90eXBlJTNBJTIwTVVMVElfR1BVJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTBBZG93bmNhc3RfYmYxNiUzQSUyMCdubyclMEFncHVfaWRzJTNBJTIwYWxsJTBBbWFjaGluZV9yYW5rJTNBJTIwMCUyMCUyMyUyMCVFQiU4NSVCOCVFQiU5MyU5QyVFQyU5NyU5MCUyMCVFQiU5NCVCMCVFQiU5RCVCQyUyMCVFQyU4OCU5QyVFQyU5QyU4NCVFQiVBNSVCQyUyMCVFQiVCMyU4MCVFQSVCMiVCRCVFRCU5NSU5OCVFQyU4NCVCOCVFQyU5QSU5NCUwQW1haW5fcHJvY2Vzc19pcCUzQSUyMDE5Mi4xNjguMjAuMSUwQW1haW5fcHJvY2Vzc19wb3J0JTNBJTIwOTg5OCUwQW1haW5fdHJhaW5pbmdfZnVuY3Rpb24lM0ElMjBtYWluJTBBbWl4ZWRfcHJlY2lzaW9uJTNBJTIwZnAxNiUwQW51bV9tYWNoaW5lcyUzQSUyMDIlMEFudW1fcHJvY2Vzc2VzJTNBJTIwOCUwQXJkenZfYmFja2VuZCUzQSUyMHN0YXRpYyUwQXNhbWVfbmV0d29yayUzQSUyMHRydWUlMEF0cHVfZW52JTNBJTIwJTVCJTVEJTBBdHB1X3VzZV9jbHVzdGVyJTNBJTIwZmFsc2UlMEF0cHVfdXNlX3N1ZG8lM0ElMjBmYWxzZSUwQXVzZV9jcHUlM0ElMjBmYWxzZQ==",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>                                                                                             
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">MULTI_GPU</span>                                                                                                    
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">gpu_ids:</span> <span class="hljs-string">all</span>
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span> <span class="hljs-comment"># 노드에 따라 순위를 변경하세요</span>
<span class="hljs-attr">main_process_ip:</span> <span class="hljs-number">192.168</span><span class="hljs-number">.20</span><span class="hljs-number">.1</span>
<span class="hljs-attr">main_process_port:</span> <span class="hljs-number">9898</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">fp16</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">2</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">8</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">tpu_env:</span> []
<span class="hljs-attr">tpu_use_cluster:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">tpu_use_sudo:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,wrap:!1}}),{c(){m(a.$$.fragment)},l(n){j(a.$$.fragment,n)},m(n,u){w(a,n,u),r=!0},p:A,i(n){r||(y(a.$$.fragment,n),r=!0)},o(n){U(a.$$.fragment,n),r=!1},d(n){d(a,n)}}}function $t(h){let a,r;return a=new g({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkaXN0cmlidXRlZF90eXBlJTNBJTIwRlNEUCUwQWRvd25jYXN0X2JmMTYlM0ElMjAnbm8nJTBBZnNkcF9jb25maWclM0ElMEElMjAlMjBmc2RwX2F1dG9fd3JhcF9wb2xpY3klM0ElMjBUUkFOU0ZPUk1FUl9CQVNFRF9XUkFQJTBBJTIwJTIwZnNkcF9iYWNrd2FyZF9wcmVmZXRjaF9wb2xpY3klM0ElMjBCQUNLV0FSRF9QUkUlMEElMjAlMjBmc2RwX2ZvcndhcmRfcHJlZmV0Y2glM0ElMjB0cnVlJTBBJTIwJTIwZnNkcF9vZmZsb2FkX3BhcmFtcyUzQSUyMGZhbHNlJTBBJTIwJTIwZnNkcF9zaGFyZGluZ19zdHJhdGVneSUzQSUyMDElMEElMjAlMjBmc2RwX3N0YXRlX2RpY3RfdHlwZSUzQSUyMEZVTExfU1RBVEVfRElDVCUwQSUyMCUyMGZzZHBfc3luY19tb2R1bGVfc3RhdGVzJTNBJTIwdHJ1ZSUwQSUyMCUyMGZzZHBfdHJhbnNmb3JtZXJfbGF5ZXJfY2xzX3RvX3dyYXAlM0ElMjBCZXJ0TGF5ZXIlMEElMjAlMjBmc2RwX3VzZV9vcmlnX3BhcmFtcyUzQSUyMHRydWUlMEFtYWNoaW5lX3JhbmslM0ElMjAwJTBBbWFpbl90cmFpbmluZ19mdW5jdGlvbiUzQSUyMG1haW4lMEFtaXhlZF9wcmVjaXNpb24lM0ElMjBiZjE2JTBBbnVtX21hY2hpbmVzJTNBJTIwMSUwQW51bV9wcm9jZXNzZXMlM0ElMjAyJTBBcmR6dl9iYWNrZW5kJTNBJTIwc3RhdGljJTBBc2FtZV9uZXR3b3JrJTNBJTIwdHJ1ZSUwQXRwdV9lbnYlM0ElMjAlNUIlNUQlMEF0cHVfdXNlX2NsdXN0ZXIlM0ElMjBmYWxzZSUwQXRwdV91c2Vfc3VkbyUzQSUyMGZhbHNlJTBBdXNlX2NwdSUzQSUyMGZhbHNl",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">FSDP</span>
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">fsdp_config:</span>
  <span class="hljs-attr">fsdp_auto_wrap_policy:</span> <span class="hljs-string">TRANSFORMER_BASED_WRAP</span>
  <span class="hljs-attr">fsdp_backward_prefetch_policy:</span> <span class="hljs-string">BACKWARD_PRE</span>
  <span class="hljs-attr">fsdp_forward_prefetch:</span> <span class="hljs-literal">true</span>
  <span class="hljs-attr">fsdp_offload_params:</span> <span class="hljs-literal">false</span>
  <span class="hljs-attr">fsdp_sharding_strategy:</span> <span class="hljs-number">1</span>
  <span class="hljs-attr">fsdp_state_dict_type:</span> <span class="hljs-string">FULL_STATE_DICT</span>
  <span class="hljs-attr">fsdp_sync_module_states:</span> <span class="hljs-literal">true</span>
  <span class="hljs-attr">fsdp_transformer_layer_cls_to_wrap:</span> <span class="hljs-string">BertLayer</span>
  <span class="hljs-attr">fsdp_use_orig_params:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">bf16</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">2</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">tpu_env:</span> []
<span class="hljs-attr">tpu_use_cluster:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">tpu_use_sudo:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,wrap:!1}}),{c(){m(a.$$.fragment)},l(n){j(a.$$.fragment,n)},m(n,u){w(a,n,u),r=!0},p:A,i(n){r||(y(a.$$.fragment,n),r=!0)},o(n){U(a.$$.fragment,n),r=!1},d(n){d(a,n)}}}function Bt(h){let a,r;return a=new g({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkZWVwc3BlZWRfY29uZmlnJTNBJTBBJTIwJTIwZGVlcHNwZWVkX2NvbmZpZ19maWxlJTNBJTIwJTJGaG9tZSUyRnVzZXIlMkZjb25maWdzJTJGZHNfemVybzNfY29uZmlnLmpzb24lMEElMjAlMjB6ZXJvM19pbml0X2ZsYWclM0ElMjB0cnVlJTBBZGlzdHJpYnV0ZWRfdHlwZSUzQSUyMERFRVBTUEVFRCUwQWRvd25jYXN0X2JmMTYlM0ElMjAnbm8nJTBBbWFjaGluZV9yYW5rJTNBJTIwMCUwQW1haW5fdHJhaW5pbmdfZnVuY3Rpb24lM0ElMjBtYWluJTBBbnVtX21hY2hpbmVzJTNBJTIwMSUwQW51bV9wcm9jZXNzZXMlM0ElMjA0JTBBcmR6dl9iYWNrZW5kJTNBJTIwc3RhdGljJTBBc2FtZV9uZXR3b3JrJTNBJTIwdHJ1ZSUwQXRwdV9lbnYlM0ElMjAlNUIlNUQlMEF0cHVfdXNlX2NsdXN0ZXIlM0ElMjBmYWxzZSUwQXRwdV91c2Vfc3VkbyUzQSUyMGZhbHNlJTBBdXNlX2NwdSUzQSUyMGZhbHNl",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>
<span class="hljs-attr">deepspeed_config:</span>
  <span class="hljs-attr">deepspeed_config_file:</span> <span class="hljs-string">/home/user/configs/ds_zero3_config.json</span>
  <span class="hljs-attr">zero3_init_flag:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">DEEPSPEED</span>
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">4</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">tpu_env:</span> []
<span class="hljs-attr">tpu_use_cluster:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">tpu_use_sudo:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,wrap:!1}}),{c(){m(a.$$.fragment)},l(n){j(a.$$.fragment,n)},m(n,u){w(a,n,u),r=!0},p:A,i(n){r||(y(a.$$.fragment,n),r=!0)},o(n){U(a.$$.fragment,n),r=!1},d(n){d(a,n)}}}function Xt(h){let a,r;return a=new g({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMEFkZWVwc3BlZWRfY29uZmlnJTNBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTBBJTIwJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTNBJTIwMSUwQSUyMCUyMGdyYWRpZW50X2NsaXBwaW5nJTNBJTIwMC43JTBBJTIwJTIwb2ZmbG9hZF9vcHRpbWl6ZXJfZGV2aWNlJTNBJTIwY3B1JTBBJTIwJTIwb2ZmbG9hZF9wYXJhbV9kZXZpY2UlM0ElMjBjcHUlMEElMjAlMjB6ZXJvM19pbml0X2ZsYWclM0ElMjB0cnVlJTBBJTIwJTIwemVyb19zdGFnZSUzQSUyMDIlMEFkaXN0cmlidXRlZF90eXBlJTNBJTIwREVFUFNQRUVEJTBBZG93bmNhc3RfYmYxNiUzQSUyMCdubyclMEFtYWNoaW5lX3JhbmslM0ElMjAwJTBBbWFpbl90cmFpbmluZ19mdW5jdGlvbiUzQSUyMG1haW4lMEFtaXhlZF9wcmVjaXNpb24lM0ElMjBiZjE2JTBBbnVtX21hY2hpbmVzJTNBJTIwMSUwQW51bV9wcm9jZXNzZXMlM0ElMjA0JTBBcmR6dl9iYWNrZW5kJTNBJTIwc3RhdGljJTBBc2FtZV9uZXR3b3JrJTNBJTIwdHJ1ZSUwQXRwdV9lbnYlM0ElMjAlNUIlNUQlMEF0cHVfdXNlX2NsdXN0ZXIlM0ElMjBmYWxzZSUwQXRwdV91c2Vfc3VkbyUzQSUyMGZhbHNlJTBBdXNlX2NwdSUzQSUyMGZhbHNl",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>                                                                                             
<span class="hljs-attr">deepspeed_config:</span>                                                                                                              
  <span class="hljs-attr">gradient_accumulation_steps:</span> <span class="hljs-number">1</span>
  <span class="hljs-attr">gradient_clipping:</span> <span class="hljs-number">0.7</span>
  <span class="hljs-attr">offload_optimizer_device:</span> <span class="hljs-string">cpu</span>
  <span class="hljs-attr">offload_param_device:</span> <span class="hljs-string">cpu</span>
  <span class="hljs-attr">zero3_init_flag:</span> <span class="hljs-literal">true</span>
  <span class="hljs-attr">zero_stage:</span> <span class="hljs-number">2</span>
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">DEEPSPEED</span>
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">bf16</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">4</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">tpu_env:</span> []
<span class="hljs-attr">tpu_use_cluster:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">tpu_use_sudo:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,wrap:!1}}),{c(){m(a.$$.fragment)},l(n){j(a.$$.fragment,n)},m(n,u){w(a,n,u),r=!0},p:A,i(n){r||(y(a.$$.fragment,n),r=!0)},o(n){U(a.$$.fragment,n),r=!1},d(n){d(a,n)}}}function Qt(h){let a,r,n,u,o,f,b,_;return a=new ql({props:{id:"config",option:"DistributedDataParallel",$$slots:{default:[Vt]},$$scope:{ctx:h}}}),n=new ql({props:{id:"config",option:"FSDP",$$slots:{default:[$t]},$$scope:{ctx:h}}}),o=new ql({props:{id:"config",option:"DeepSpeed",$$slots:{default:[Bt]},$$scope:{ctx:h}}}),b=new ql({props:{id:"config",option:"DeepSpeed with Accelerate plugin",$$slots:{default:[Xt]},$$scope:{ctx:h}}}),{c(){m(a.$$.fragment),r=M(),m(n.$$.fragment),u=M(),m(o.$$.fragment),f=M(),m(b.$$.fragment)},l(J){j(a.$$.fragment,J),r=p(J),j(n.$$.fragment,J),u=p(J),j(o.$$.fragment,J),f=p(J),j(b.$$.fragment,J)},m(J,I){w(a,J,I),t(J,r,I),w(n,J,I),t(J,u,I),w(o,J,I),t(J,f,I),w(b,J,I),_=!0},p(J,I){const F={};I&2&&(F.$$scope={dirty:I,ctx:J}),a.$set(F);const C={};I&2&&(C.$$scope={dirty:I,ctx:J}),n.$set(C);const Pl={};I&2&&(Pl.$$scope={dirty:I,ctx:J}),o.$set(Pl);const E={};I&2&&(E.$$scope={dirty:I,ctx:J}),b.$set(E)},i(J){_||(y(a.$$.fragment,J),y(n.$$.fragment,J),y(o.$$.fragment,J),y(b.$$.fragment,J),_=!0)},o(J){U(a.$$.fragment,J),U(n.$$.fragment,J),U(o.$$.fragment,J),U(b.$$.fragment,J),_=!1},d(J){J&&(e(r),e(u),e(f)),d(a,J),d(n,J),d(o,J),d(b,J)}}}function Rt(h){let a,r,n,u,o,f,b,_="<code>Trainer</code>는 Transformers 라이브러리에 구현된 PyTorch 모델을 반복하여 훈련 및 평가 과정입니다. 훈련에 필요한 요소(모델, 토크나이저, 데이터셋, 평가 함수, 훈련 하이퍼파라미터 등)만 제공하면 <code>Trainer</code>가 필요한 나머지 작업을 처리합니다. 이를 통해 직접 훈련 루프를 작성하지 않고도 빠르게 훈련을 시작할 수 있습니다. 또한 <code>Trainer</code>는 강력한 맞춤 설정과 다양한 훈련 옵션을 제공하여 사용자 맞춤 훈련이 가능합니다.",J,I,F,C,Pl='시작하기 전에, 분산 환경에서 PyTorch 훈련과 실행을 할 수 있게 <a href="https://hf.co/docs/accelerate" rel="nofollow">Accelerate</a> 라이브러리가 설치되었는지 확인하세요.',E,W,ls,N,de="이 가이드는 <code>Trainer</code> 클래스에 대한 개요를 제공합니다.",ss,G,es,k,ue="<code>Trainer</code>는 기본적인 훈련 루프에 필요한 모든 코드를 포함하고 있습니다.",ts,v,fe="<li>손실을 계산하는 훈련 단계를 수행합니다.</li> <li><code>backward</code> 메소드로 그레이디언트를 계산합니다.</li> <li>그레이디언트를 기반으로 가중치를 업데이트합니다.</li> <li>정해진 에폭 수에 도달할 때까지 이 과정을 반복합니다.</li>",as,S,he="<code>Trainer</code> 클래스는 PyTorch와 훈련 과정에 익숙하지 않거나 막 시작한 경우에도 훈련이 가능하도록 필요한 모든 코드를 추상화하였습니다. 또한 매번 훈련 루프를 손수 작성하지 않아도 되며, 훈련에 필요한 모델과 데이터셋 같은 필수 구성 요소만 제공하면, [Trainer] 클래스가 나머지를 처리합니다.",ns,Y,be="훈련 옵션이나 하이퍼파라미터를 지정하려면, <code>TrainingArguments</code> 클래스에서 확인 할 수 있습니다. 예를 들어, 모델을 저장할 디렉토리를 <code>output_dir</code>에 정의하고, 훈련 후에 Hub로 모델을 푸시하려면 <code>push_to_hub=True</code>로 설정합니다.",Ms,H,ps,z,Ie="<code>training_args</code>를 <code>Trainer</code>에 모델, 데이터셋, 데이터셋 전처리 도구(데이터 유형에 따라 토크나이저, 특징 추출기 또는 이미지 프로세서일 수 있음), 데이터 수집기 및 훈련 중 확인할 지표를 계산할 함수를 함께 전달하세요.",rs,x,_e="마지막으로, <code>train()</code>를 호출하여 훈련을 시작하세요!",cs,L,is,D,Ts,q,ge="<code>Trainer</code> 클래스는 <code>TrainingArguments</code>의 <code>output_dir</code> 매개변수에 지정된 디렉토리에 모델 체크포인트를 저장합니다. 체크포인트는 <code>checkpoint-000</code> 하위 폴더에 저장되며, 여기서 끝의 숫자는 훈련 단계에 해당합니다. 체크포인트를 저장하면 나중에 훈련을 재개할 때 유용합니다.",os,P,Js,O,Ae='체크포인트를 Hub에 푸시하려면 <code>TrainingArguments</code>에서 <code>push_to_hub=True</code>로 설정하여 커밋하고 푸시할 수 있습니다. 체크포인트 저장 방법을 결정하는 다른 옵션은 <a href="https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments.hub_strategy" rel="nofollow"><code>hub_strategy</code></a> 매개변수에서 설정합니다:',ms,K,Ce="<li><code>hub_strategy=&quot;checkpoint&quot;</code>는 최신 체크포인트를 “last-checkpoint”라는 하위 폴더에 푸시하여 훈련을 재개할 수 있습니다.</li> <li><code>hub_strategy=&quot;all_checkpoints&quot;</code>는 모든 체크포인트를 <code>output_dir</code>에 정의된 디렉토리에 푸시합니다(모델 리포지토리에서 폴더당 하나의 체크포인트를 볼 수 있습니다).</li>",js,ll,Ze='체크포인트에서 훈련을 재개할 때, <code>Trainer</code>는 체크포인트가 저장될 때와 동일한 Python, NumPy 및 PyTorch RNG 상태를 유지하려고 합니다. 하지만 PyTorch는 기본 설정으로 ‘일관된 결과를 보장하지 않음’으로 많이 되어있기 때문에, RNG 상태가 동일할 것이라고 보장할 수 없습니다. 따라서, 일관된 결과가 보장되도록 활성화 하려면, <a href="https://pytorch.org/docs/stable/notes/randomness#controlling-sources-of-randomness" rel="nofollow">랜덤성 제어</a> 가이드를 참고하여 훈련을 완전히 일관된 결과를 보장 받도록 만들기 위해 활성화할 수 있는 항목을 확인하세요. 다만, 특정 설정을 결정적으로 만들면 훈련이 느려질 수 있습니다.',ws,sl,ys,el,Ve="<code>Trainer</code> 클래스는 접근성과 용이성을 염두에 두고 설계되었지만, 더 다양한 기능을 원하는 사용자들을 위해 다양한 맞춤 설정 옵션을 제공합니다. <code>Trainer</code>의 많은 메소드는 서브클래스화 및 오버라이드하여 원하는 기능을 제공할 수 있으며, 이를 통해 전체 훈련 루프를 다시 작성할 필요 없이 원하는 기능을 추가할 수 있습니다. 이러한 메소드에는 다음이 포함됩니다:",Us,tl,$e="<li><code>get_train_dataloader()</code>는 훈련 데이터로더를 생성합니다.</li> <li><code>get_eval_dataloader()</code>는 평가 데이터로더를 생성합니다.</li> <li><code>get_test_dataloader()</code>는 테스트 데이터로더를 생성합니다.</li> <li><code>log()</code>는 훈련을 모니터링하는 다양한 객체에 대한 정보를 로그로 남깁니다.</li> <li><code>create_optimizer_and_scheduler()</code>는 <code>__init__</code>에서 전달되지 않은 경우 옵티마이저와 학습률 스케줄러를 생성합니다. 이들은 각각 <code>create_optimizer()</code> 및 <code>create_scheduler()</code>로 별도로 맞춤 설정 할 수 있습니다.</li> <li><code>compute_loss()</code>는 훈련 입력 배치에 대한 손실을 계산합니다.</li> <li><code>training_step()</code>는 훈련 단계를 수행합니다.</li> <li><code>prediction_step()</code>는 예측 및 테스트 단계를 수행합니다.</li> <li><code>evaluate()</code>는 모델을 평가하고 평가 지표을 반환합니다.</li> <li><code>predict()</code>는 테스트 세트에 대한 예측(레이블이 있는 경우 지표 포함)을 수행합니다.</li>",ds,al,Be="예를 들어, <code>compute_loss()</code> 메소드를 맞춤 설정하여 가중 손실을 사용하려는 경우:",us,nl,fs,Ml,hs,pl,Xe='<code>Trainer</code>를 맞춤 설정하는 또 다른 방법은 <a href="callbacks">콜백</a>을 사용하는 것입니다. 콜백은 훈련 루프에서 <em>변화를 주지 않습니다</em>. 훈련 루프의 상태를 검사한 후 상태에 따라 일부 작업(조기 종료, 결과 로그 등)을 실행합니다. 즉, 콜백은 사용자 정의 손실 함수와 같은 것을 구현하는 데 사용할 수 없으며, 이를 위해서는 <code>compute_loss()</code> 메소드를 서브클래스화하고 오버라이드해야 합니다.',bs,rl,Qe="예를 들어, 훈련 루프에 10단계 후 조기 종료 콜백을 추가하려면 다음과 같이 합니다.",Is,cl,_s,il,Re="그런 다음, 이를 <code>Trainer</code>의 <code>callback</code> 매개변수에 전달합니다.",gs,Tl,As,ol,Cs,V,Zs,Jl,Fe='<code>Trainer</code>는 기본적으로 <code>logging.INFO</code>로 설정되어 있어 오류, 경고 및 기타 기본 정보를 보고합니다. 분산 환경에서는 <code>Trainer</code> 복제본이 <code>logging.WARNING</code>으로 설정되어 오류와 경고만 보고합니다. <code>TrainingArguments</code>의 <a href="https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments.log_level" rel="nofollow"><code>log_level</code></a> 및 <a href="https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments.log_level_replica" rel="nofollow"><code>log_level_replica</code></a> 매개변수로 로그 레벨을 변경할 수 있습니다.',Vs,ml,Ee='각 노드의 로그 레벨 설정을 구성하려면 <a href="https://huggingface.co/docs/transformers/main/en/main_classes/trainer#transformers.TrainingArguments.log_on_each_node" rel="nofollow"><code>log_on_each_node</code></a> 매개변수를 사용하여 각 노드에서 로그 레벨을 사용할지 아니면 주 노드에서만 사용할지 결정하세요.',$s,$,Bs,jl,We="예를 들어, 메인 코드와 모듈을 각 노드에 따라 동일한 로그 레벨을 사용하도록 설정하려면 다음과 같이 합니다.",Xs,wl,Qs,yl,Ne="각 노드에서 기록될 내용을 구성하기 위해 <code>log_level</code>과 <code>log_level_replica</code>를 다양한 조합으로 사용해보세요.",Rs,B,Fs,Ul,Es,dl,Ge='<a href="https://hf.co/papers/2310.05914" rel="nofollow">NEFTune</a>은 훈련 중 임베딩 벡터에 노이즈를 추가하여 성능을 향상시킬 수 있는 기술입니다. <code>Trainer</code>에서 이를 활성화하려면 <code>TrainingArguments</code>의 <code>neftune_noise_alpha</code> 매개변수를 설정하여 노이즈의 양을 조절합니다.',Ws,ul,Ns,fl,ke="NEFTune은 예상치 못한 동작을 피할 목적으로 처음 임베딩 레이어로 복원하기 위해 훈련 후 비활성화 됩니다.",Gs,hl,ks,bl,ve="Gradient Low-Rank Projection (GaLore)은 전체 매개변수를 학습하면서도 LoRA와 같은 일반적인 저계수 적응 방법보다 더 메모리 효율적인 저계수 학습 전략입니다.",vs,Il,Se="먼저 GaLore 공식 리포지토리를 설치합니다:",Ss,_l,Ys,gl,Ye="그런 다음 <code>optim</code>에 <code>[&quot;galore_adamw&quot;, &quot;galore_adafactor&quot;, &quot;galore_adamw_8bit&quot;]</code> 중 하나와 함께 <code>optim_target_modules</code>를 추가합니다. 이는 적용하려는 대상 모듈 이름에 해당하는 문자열, 정규 표현식 또는 전체 경로의 목록일 수 있습니다. 아래는 end-to-end 예제 스크립트입니다(필요한 경우 <code>pip install trl datasets</code>를 실행):",Hs,Al,zs,Cl,He="GaLore가 지원하는 추가 매개변수를 전달하려면 <code>optim_args</code>를 설정합니다. 예를 들어:",xs,Zl,Ls,Vl,ze='해당 방법에 대한 자세한 내용은 <a href="https://github.com/jiaweizzhao/GaLore" rel="nofollow">원본 리포지토리</a> 또는 <a href="https://arxiv.org/abs/2403.03507" rel="nofollow">논문</a>을 참고하세요.',Ds,$l,xe="현재 GaLore 레이어로 간주되는 Linear 레이어만 훈련 할수 있으며, 저계수 분해를 사용하여 훈련되고 나머지 레이어는 기존 방식으로 최적화됩니다.",qs,Bl,Le="훈련 시작 전에 시간이 약간 걸릴 수 있습니다(NVIDIA A100에서 2B 모델의 경우 약 3분), 하지만 이후 훈련은 원활하게 진행됩니다.",Ps,Xl,De="다음과 같이 옵티마이저 이름에 <code>layerwise</code>를 추가하여 레이어별 최적화를 수행할 수도 있습니다:",Os,Ql,Ks,Rl,qe='레이어별 최적화는 다소 실험적이며 DDP(분산 데이터 병렬)를 지원하지 않으므로, 단일 GPU에서만 훈련 스크립트를 실행할 수 있습니다. 자세한 내용은 <a href="https://github.com/jiaweizzhao/GaLore?tab=readme-ov-file#train-7b-model-with-a-single-gpu-with-24gb-memory" rel="nofollow">이 문서를</a>을 참조하세요. gradient clipping, DeepSpeed 등 다른 기능은 기본적으로 지원되지 않을 수 있습니다. 이러한 문제가 발생하면 <a href="https://github.com/huggingface/transformers/issues" rel="nofollow">GitHub에 이슈를 올려주세요</a>.',le,Fl,se,El,Pe=`LOMO 옵티마이저는 <a href="https://hf.co/papers/2306.09782" rel="nofollow">제한된 자원으로 대형 언어 모델의 전체 매개변수 미세 조정</a>과 <a href="https://hf.co/papers/2310.10195" rel="nofollow">적응형 학습률을 통한 저메모리 최적화(AdaLomo)</a>에서 도입되었습니다.
이들은 모두 효율적인 전체 매개변수 미세 조정 방법으로 구성되어 있습니다. 이러한 옵티마이저들은 메모리 사용량을 줄이기 위해 그레이디언트 계산과 매개변수 업데이트를 하나의 단계로 융합합니다. LOMO에서 지원되는 옵티마이저는 <code>&quot;lomo&quot;</code>와 <code>&quot;adalomo&quot;</code>입니다. 먼저 pypi에서 <code>pip install lomo-optim</code>를 통해 <code>lomo</code>를 설치하거나, GitHub 소스에서 <code>pip install git+https://github.com/OpenLMLab/LOMO.git</code>로 설치하세요.`,ee,X,te,Wl,Oe='다음은 IMDB 데이터셋에서 <a href="https://huggingface.co/google/gemma-2b" rel="nofollow">google/gemma-2b</a>를 최대 정밀도로 미세 조정하는 간단한 스크립트입니다:',ae,Nl,ne,Gl,Me,kl,Ke='<code>Trainer</code> 클래스는 <a href="https://hf.co/docs/accelerate" rel="nofollow">Accelerate</a>로 구동되며, 이는 <a href="https://pytorch.org/blog/introducing-pytorch-fully-sharded-data-parallel-api/" rel="nofollow">FullyShardedDataParallel (FSDP)</a> 및 <a href="https://www.deepspeed.ai/" rel="nofollow">DeepSpeed</a>와 같은 통합을 지원하는 분산 환경에서 PyTorch 모델을 쉽게 훈련할 수 있는 라이브러리입니다.',pe,Q,re,vl,lt='<code>Trainer</code>와 Accelerate를 사용하려면 <a href="https://huggingface.co/docs/accelerate/package_reference/cli#accelerate-config" rel="nofollow"><code>accelerate.config</code></a> 명령을 실행하여 훈련 환경을 설정하세요. 이 명령은 훈련 스크립트를 실행할 때 사용할 <code>config_file.yaml</code>을 생성합니다. 예를 들어, 다음 예시는 설정할 수 있는 일부 구성 예입니다.',ce,R,ie,Sl,st='<a href="https://huggingface.co/docs/accelerate/package_reference/cli#accelerate-launch" rel="nofollow"><code>accelerate_launch</code></a> 명령은 Accelerate와 <code>Trainer</code>를 사용하여 분산 시스템에서 훈련 스크립트를 실행하는 권장 방법이며, <code>config_file.yaml</code>에 지정된 매개변수를 사용합니다. 이 파일은 Accelerate 캐시 폴더에 저장되며 <code>accelerate_launch</code>를 실행할 때 자동으로 로드됩니다.',Te,Yl,et='예를 들어, FSDP 구성을 사용하여 <a href="https://github.com/huggingface/transformers/blob/f4db565b695582891e43a5e042e5d318e28f20b8/examples/pytorch/text-classification/run_glue.py#L4" rel="nofollow">run_glue.py</a> 훈련 스크립트를 실행하려면 다음과 같이 합니다:',oe,Hl,Je,zl,tt="<code>config_file.yaml</code> 파일의 매개변수를 직접 지정할 수도 있습니다:",me,xl,je,Ll,at='<code>accelerate_launch</code>와 사용자 정의 구성에 대해 더 알아보려면 <a href="https://huggingface.co/docs/accelerate/basic_tutorials/launch" rel="nofollow">Accelerate 스크립트 실행</a> 튜토리얼을 확인하세요.',we,Dl,ye,Ol,Ue;return o=new Z({props:{title:"Trainer",local:"trainer",headingTag:"h1"}}),I=new Kl({props:{$$slots:{default:[ht]},$$scope:{ctx:h}}}),W=new g({props:{code:"cGlwJTIwaW5zdGFsbCUyMGFjY2VsZXJhdGUlMEElMEElMjMlMjAlRUMlOTclODUlRUElQjclQjglRUIlQTAlODglRUMlOUQlQjQlRUIlOTMlOUMlMEFwaXAlMjBpbnN0YWxsJTIwYWNjZWxlcmF0ZSUyMC0tdXBncmFkZQ==",highlighted:`pip install accelerate

<span class="hljs-comment"># 업그레이드</span>
pip install accelerate --upgrade`,wrap:!1}}),G=new Z({props:{title:"기본 사용법",local:"basic-usage",headingTag:"h2"}}),H=new g({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluaW5nQXJndW1lbnRzJTBBJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJ5b3VyLW1vZGVsJTIyJTJDJTBBJTIwJTIwJTIwJTIwbGVhcm5pbmdfcmF0ZSUzRDJlLTUlMkMlMEElMjAlMjAlMjAlMjBwZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfZXZhbF9iYXRjaF9zaXplJTNEMTYlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fZXBvY2hzJTNEMiUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodF9kZWNheSUzRDAuMDElMkMlMEElMjAlMjAlMjAlMjBldmFsX3N0cmF0ZWd5JTNEJTIyZXBvY2glMjIlMkMlMEElMjAlMjAlMjAlMjBzYXZlX3N0cmF0ZWd5JTNEJTIyZXBvY2glMjIlMkMlMEElMjAlMjAlMjAlMjBsb2FkX2Jlc3RfbW9kZWxfYXRfZW5kJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHB1c2hfdG9faHViJTNEVHJ1ZSUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments

training_args = TrainingArguments(
    output_dir=<span class="hljs-string">&quot;your-model&quot;</span>,
    learning_rate=<span class="hljs-number">2e-5</span>,
    per_device_train_batch_size=<span class="hljs-number">16</span>,
    per_device_eval_batch_size=<span class="hljs-number">16</span>,
    num_train_epochs=<span class="hljs-number">2</span>,
    weight_decay=<span class="hljs-number">0.01</span>,
    eval_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
    save_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
    load_best_model_at_end=<span class="hljs-literal">True</span>,
    push_to_hub=<span class="hljs-literal">True</span>,
)`,wrap:!1}}),L=new g({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluZXIlMEElMEF0cmFpbmVyJTIwJTNEJTIwVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZGF0YXNldCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEZGF0YXNldCU1QiUyMnRlc3QlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMEElMjAlMjAlMjAlMjBkYXRhX2NvbGxhdG9yJTNEZGF0YV9jb2xsYXRvciUyQyUwQSUyMCUyMCUyMCUyMGNvbXB1dGVfbWV0cmljcyUzRGNvbXB1dGVfbWV0cmljcyUyQyUwQSklMEElMEF0cmFpbmVyLnRyYWluKCk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=dataset[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=dataset[<span class="hljs-string">&quot;test&quot;</span>],
    tokenizer=tokenizer,
    data_collator=data_collator,
    compute_metrics=compute_metrics,
)

trainer.train()`,wrap:!1}}),D=new Z({props:{title:"체크포인트",local:"checkpoints",headingTag:"h3"}}),P=new g({props:{code:"JTIzJTIwJUVDJUI1JTlDJUVDJThCJUEwJTIwJUVDJUIyJUI0JUVEJTgxJUFDJUVEJThGJUFDJUVDJTlEJUI4JUVEJThBJUI4JUVDJTk3JTkwJUVDJTg0JTlDJTIwJUVDJTlFJUFDJUVBJUIwJTlDJTBBdHJhaW5lci50cmFpbihyZXN1bWVfZnJvbV9jaGVja3BvaW50JTNEVHJ1ZSklMEElMEElMjMlMjAlRUMlQjYlOUMlRUIlQTAlQTUlMjAlRUIlOTQlOTQlRUIlQTAlODklRUQlODYlQTAlRUIlQTYlQUMlRUMlOTclOTAlMjAlRUMlQTAlODAlRUMlOUUlQTUlRUIlOTAlOUMlMjAlRUQlOEElQjklRUMlQTAlOTUlMjAlRUMlQjIlQjQlRUQlODElQUMlRUQlOEYlQUMlRUMlOUQlQjglRUQlOEElQjglRUMlOTclOTAlRUMlODQlOUMlMjAlRUMlOUUlQUMlRUElQjAlOUMlMEF0cmFpbmVyLnRyYWluKHJlc3VtZV9mcm9tX2NoZWNrcG9pbnQlM0QlMjJ5b3VyLW1vZGVsJTJGY2hlY2twb2ludC0xMDAwJTIyKQ==",highlighted:`<span class="hljs-comment"># 최신 체크포인트에서 재개</span>
trainer.train(resume_from_checkpoint=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># 출력 디렉토리에 저장된 특정 체크포인트에서 재개</span>
trainer.train(resume_from_checkpoint=<span class="hljs-string">&quot;your-model/checkpoint-1000&quot;</span>)`,wrap:!1}}),sl=new Z({props:{title:"Trainer 맞춤 설정",local:"customize-the-trainer",headingTag:"h2"}}),nl=new g({props:{code:"ZnJvbSUyMHRvcmNoJTIwaW1wb3J0JTIwbm4lMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwVHJhaW5lciUwQSUwQWNsYXNzJTIwQ3VzdG9tVHJhaW5lcihUcmFpbmVyKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGNvbXB1dGVfbG9zcyhzZWxmJTJDJTBBJTBBJTIwbW9kZWwlMkMlMjBpbnB1dHMlMkMlMjByZXR1cm5fb3V0cHV0cyUzREZhbHNlKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxhYmVscyUyMCUzRCUyMGlucHV0cy5wb3AoJTIybGFiZWxzJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMCVFQyU4OCU5QyVFQiVCMCVBOSVFRCU5NiVBNSUyMCVFQyVBMCU4NCVFRCU4QyU4QyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb2dpdHMlMjAlM0QlMjBvdXRwdXRzLmdldCglMjJsb2dpdHMlMjIpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwJUVDJTg0JTlDJUVCJUExJTlDJTIwJUVCJThCJUE0JUVCJUE1JUI4JTIwJUVBJUIwJTgwJUVDJUE0JTkxJUVDJUI5JTk4JUVCJUExJTlDJTIwMyVFQSVCMCU5QyVFQyU5RCU5OCUyMCVFQiVBMCU4OCVFQyU5RCVCNCVFQiVCOCU5NCVFQyU5NyU5MCUyMCVFQiU4QyU4MCVFRCU5NSU5QyUyMCVFQyU4MiVBQyVFQyU5QSVBOSVFQyU5RSU5MCUyMCVFQyVBMCU5NSVFQyU5RCU5OCUyMCVFQyU4NiU5MCVFQyU4QiVBNCVFQyU5RCU4NCUyMCVFQSVCMyU4NCVFQyU4MiVCMCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxvc3NfZmN0JTIwJTNEJTIwbm4uQ3Jvc3NFbnRyb3B5TG9zcyh3ZWlnaHQlM0R0b3JjaC50ZW5zb3IoJTVCMS4wJTJDJTIwMi4wJTJDJTIwMy4wJTVEJTJDJTIwZGV2aWNlJTNEbW9kZWwuZGV2aWNlKSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zc19mY3QobG9naXRzLnZpZXcoLTElMkMlMjBzZWxmLm1vZGVsLmNvbmZpZy5udW1fbGFiZWxzKSUyQyUyMGxhYmVscy52aWV3KC0xKSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjAobG9zcyUyQyUyMG91dHB1dHMpJTIwaWYlMjByZXR1cm5fb3V0cHV0cyUyMGVsc2UlMjBsb3Nz",highlighted:`<span class="hljs-keyword">from</span> torch <span class="hljs-keyword">import</span> nn
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer

<span class="hljs-keyword">class</span> <span class="hljs-title class_">CustomTrainer</span>(<span class="hljs-title class_ inherited__">Trainer</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_loss</span>(<span class="hljs-params">self,

 model, inputs, return_outputs=<span class="hljs-literal">False</span></span>):
        labels = inputs.pop(<span class="hljs-string">&quot;labels&quot;</span>)
        <span class="hljs-comment"># 순방향 전파</span>
        outputs = model(**inputs)
        logits = outputs.get(<span class="hljs-string">&quot;logits&quot;</span>)
        <span class="hljs-comment"># 서로 다른 가중치로 3개의 레이블에 대한 사용자 정의 손실을 계산</span>
        loss_fct = nn.CrossEntropyLoss(weight=torch.tensor([<span class="hljs-number">1.0</span>, <span class="hljs-number">2.0</span>, <span class="hljs-number">3.0</span>], device=model.device))
        loss = loss_fct(logits.view(-<span class="hljs-number">1</span>, self.model.config.num_labels), labels.view(-<span class="hljs-number">1</span>))
        <span class="hljs-keyword">return</span> (loss, outputs) <span class="hljs-keyword">if</span> return_outputs <span class="hljs-keyword">else</span> loss`,wrap:!1}}),Ml=new Z({props:{title:"콜백",local:"callbacks",headingTag:"h3"}}),cl=new g({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluZXJDYWxsYmFjayUwQSUwQWNsYXNzJTIwRWFybHlTdG9wcGluZ0NhbGxiYWNrKFRyYWluZXJDYWxsYmFjayklM0ElMEElMjAlMjAlMjAlMjBkZWYlMjBfX2luaXRfXyhzZWxmJTJDJTIwbnVtX3N0ZXBzJTNEMTApJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5udW1fc3RlcHMlMjAlM0QlMjBudW1fc3RlcHMlMEElMjAlMjAlMjAlMjAlMEElMjAlMjAlMjAlMjBkZWYlMjBvbl9zdGVwX2VuZChzZWxmJTJDJTIwYXJncyUyQyUyMHN0YXRlJTJDJTIwY29udHJvbCUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwc3RhdGUuZ2xvYmFsX3N0ZXAlMjAlM0UlM0QlMjBzZWxmLm51bV9zdGVwcyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMCU3QiUyMnNob3VsZF90cmFpbmluZ19zdG9wJTIyJTNBJTIwVHJ1ZSU3RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjAlN0IlN0Q=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainerCallback

<span class="hljs-keyword">class</span> <span class="hljs-title class_">EarlyStoppingCallback</span>(<span class="hljs-title class_ inherited__">TrainerCallback</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, num_steps=<span class="hljs-number">10</span></span>):
        self.num_steps = num_steps
    
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">on_step_end</span>(<span class="hljs-params">self, args, state, control, **kwargs</span>):
        <span class="hljs-keyword">if</span> state.global_step &gt;= self.num_steps:
            <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;should_training_stop&quot;</span>: <span class="hljs-literal">True</span>}
        <span class="hljs-keyword">else</span>:
            <span class="hljs-keyword">return</span> {}`,wrap:!1}}),Tl=new g({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluZXIlMEElMEF0cmFpbmVyJTIwJTNEJTIwVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZGF0YXNldCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEZGF0YXNldCU1QiUyMnRlc3QlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMEElMjAlMjAlMjAlMjBkYXRhX2NvbGxhdG9yJTNEZGF0YV9jb2xsYXRvciUyQyUwQSUyMCUyMCUyMCUyMGNvbXB1dGVfbWV0cmljcyUzRGNvbXB1dGVfbWV0cmljcyUyQyUwQSUyMCUyMCUyMCUyMGNhbGxiYWNrcyUzRCU1QkVhcmx5U3RvcHBpbmdDYWxsYmFjaygpJTVEJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=dataset[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=dataset[<span class="hljs-string">&quot;test&quot;</span>],
    tokenizer=tokenizer,
    data_collator=data_collator,
    compute_metrics=compute_metrics,
    callbacks=[EarlyStoppingCallback()],
)`,wrap:!1}}),ol=new Z({props:{title:"로깅",local:"logging",headingTag:"h2"}}),V=new Kl({props:{$$slots:{default:[bt]},$$scope:{ctx:h}}}),$=new Kl({props:{$$slots:{default:[It]},$$scope:{ctx:h}}}),wl=new g({props:{code:"bG9nZ2VyJTIwJTNEJTIwbG9nZ2luZy5nZXRMb2dnZXIoX19uYW1lX18pJTBBJTBBbG9nZ2luZy5iYXNpY0NvbmZpZyglMEElMjAlMjAlMjAlMjBmb3JtYXQlM0QlMjIlMjUoYXNjdGltZSlzJTIwLSUyMCUyNShsZXZlbG5hbWUpcyUyMC0lMjAlMjUobmFtZSlzJTIwLSUyMCUyNShtZXNzYWdlKXMlMjIlMkMlMEElMjAlMjAlMjAlMjBkYXRlZm10JTNEJTIyJTI1bSUyRiUyNWQlMkYlMjVZJTIwJTI1SCUzQSUyNU0lM0ElMjVTJTIyJTJDJTBBJTIwJTIwJTIwJTIwaGFuZGxlcnMlM0QlNUJsb2dnaW5nLlN0cmVhbUhhbmRsZXIoc3lzLnN0ZG91dCklNUQlMkMlMEEpJTBBJTBBbG9nX2xldmVsJTIwJTNEJTIwdHJhaW5pbmdfYXJncy5nZXRfcHJvY2Vzc19sb2dfbGV2ZWwoKSUwQWxvZ2dlci5zZXRMZXZlbChsb2dfbGV2ZWwpJTBBZGF0YXNldHMudXRpbHMubG9nZ2luZy5zZXRfdmVyYm9zaXR5KGxvZ19sZXZlbCklMEF0cmFuc2Zvcm1lcnMudXRpbHMubG9nZ2luZy5zZXRfdmVyYm9zaXR5KGxvZ19sZXZlbCklMEElMEF0cmFpbmVyJTIwJTNEJTIwVHJhaW5lciguLi4p",highlighted:`logger = logging.getLogger(__name__)

logging.basicConfig(
    <span class="hljs-built_in">format</span>=<span class="hljs-string">&quot;%(asctime)s - %(levelname)s - %(name)s - %(message)s&quot;</span>,
    datefmt=<span class="hljs-string">&quot;%m/%d/%Y %H:%M:%S&quot;</span>,
    handlers=[logging.StreamHandler(sys.stdout)],
)

log_level = training_args.get_process_log_level()
logger.setLevel(log_level)
datasets.utils.logging.set_verbosity(log_level)
transformers.utils.logging.set_verbosity(log_level)

trainer = Trainer(...)`,wrap:!1}}),B=new mt({props:{id:"logging",options:["single node","multi-node"],$$slots:{default:[At]},$$scope:{ctx:h}}}),Ul=new Z({props:{title:"NEFTune",local:"neftune",headingTag:"h2"}}),ul=new g({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluaW5nQXJndW1lbnRzJTJDJTIwVHJhaW5lciUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBUcmFpbmluZ0FyZ3VtZW50cyguLi4lMkMlMjBuZWZ0dW5lX25vaXNlX2FscGhhJTNEMC4xKSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKC4uLiUyQyUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, Trainer

training_args = TrainingArguments(..., neftune_noise_alpha=<span class="hljs-number">0.1</span>)
trainer = Trainer(..., args=training_args)`,wrap:!1}}),hl=new Z({props:{title:"GaLore",local:"galore",headingTag:"h2"}}),_l=new g({props:{code:"cGlwJTIwaW5zdGFsbCUyMGdhbG9yZS10b3JjaA==",highlighted:"pip install galore-torch",wrap:!1}}),Al=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjBkYXRhc2V0cyUwQWltcG9ydCUyMHRybCUwQSUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUcmFpbmluZ0FyZ3VtZW50cyUyQyUyMEF1dG9Db25maWclMkMlMjBBdXRvVG9rZW5pemVyJTJDJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEF0cmFpbl9kYXRhc2V0JTIwJTNEJTIwZGF0YXNldHMubG9hZF9kYXRhc2V0KCdpbWRiJyUyQyUyMHNwbGl0JTNEJ3RyYWluJyklMEElMEFhcmdzJTIwJTNEJTIwVHJhaW5pbmdBcmd1bWVudHMoJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMi4lMkZ0ZXN0LWdhbG9yZSUyMiUyQyUwQSUyMCUyMCUyMCUyMG1heF9zdGVwcyUzRDEwMCUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDIlMkMlMEElMjAlMjAlMjAlMjBvcHRpbSUzRCUyMmdhbG9yZV9hZGFtdyUyMiUyQyUwQSUyMCUyMCUyMCUyMG9wdGltX3RhcmdldF9tb2R1bGVzJTNEJTVCJTIyYXR0biUyMiUyQyUyMCUyMm1scCUyMiU1RCUwQSklMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMmdvb2dsZSUyRmdlbW1hLTJiJTIyJTBBJTBBY29uZmlnJTIwJTNEJTIwQXV0b0NvbmZpZy5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX2NvbmZpZyhjb25maWcpLnRvKDApJTBBJTBBdHJhaW5lciUyMCUzRCUyMHRybC5TRlRUcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWwlMkMlMjAlMEElMjAlMjAlMjAlMjBhcmdzJTNEYXJncyUyQyUwQSUyMCUyMCUyMCUyMHRyYWluX2RhdGFzZXQlM0R0cmFpbl9kYXRhc2V0JTJDJTBBJTIwJTIwJTIwJTIwZGF0YXNldF90ZXh0X2ZpZWxkJTNEJ3RleHQnJTJDJTBBJTIwJTIwJTIwJTIwbWF4X3NlcV9sZW5ndGglM0Q1MTIlMkMlMEEpJTBBJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> datasets
<span class="hljs-keyword">import</span> trl

<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, AutoConfig, AutoTokenizer, AutoModelForCausalLM

train_dataset = datasets.load_dataset(<span class="hljs-string">&#x27;imdb&#x27;</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>)

args = TrainingArguments(
    output_dir=<span class="hljs-string">&quot;./test-galore&quot;</span>,
    max_steps=<span class="hljs-number">100</span>,
    per_device_train_batch_size=<span class="hljs-number">2</span>,
    optim=<span class="hljs-string">&quot;galore_adamw&quot;</span>,
    optim_target_modules=[<span class="hljs-string">&quot;attn&quot;</span>, <span class="hljs-string">&quot;mlp&quot;</span>]
)

model_id = <span class="hljs-string">&quot;google/gemma-2b&quot;</span>

config = AutoConfig.from_pretrained(model_id)

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_config(config).to(<span class="hljs-number">0</span>)

trainer = trl.SFTTrainer(
    model=model, 
    args=args,
    train_dataset=train_dataset,
    dataset_text_field=<span class="hljs-string">&#x27;text&#x27;</span>,
    max_seq_length=<span class="hljs-number">512</span>,
)

trainer.train()`,wrap:!1}}),Zl=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjBkYXRhc2V0cyUwQWltcG9ydCUyMHRybCUwQSUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUcmFpbmluZ0FyZ3VtZW50cyUyQyUyMEF1dG9Db25maWclMkMlMjBBdXRvVG9rZW5pemVyJTJDJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEF0cmFpbl9kYXRhc2V0JTIwJTNEJTIwZGF0YXNldHMubG9hZF9kYXRhc2V0KCdpbWRiJyUyQyUyMHNwbGl0JTNEJ3RyYWluJyklMEElMEFhcmdzJTIwJTNEJTIwVHJhaW5pbmdBcmd1bWVudHMoJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMi4lMkZ0ZXN0LWdhbG9yZSUyMiUyQyUwQSUyMCUyMCUyMCUyMG1heF9zdGVwcyUzRDEwMCUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDIlMkMlMEElMjAlMjAlMjAlMjBvcHRpbSUzRCUyMmdhbG9yZV9hZGFtdyUyMiUyQyUwQSUyMCUyMCUyMCUyMG9wdGltX3RhcmdldF9tb2R1bGVzJTNEJTVCJTIyYXR0biUyMiUyQyUyMCUyMm1scCUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMG9wdGltX2FyZ3MlM0QlMjJyYW5rJTNENjQlMkMlMjB1cGRhdGVfcHJval9nYXAlM0QxMDAlMkMlMjBzY2FsZSUzRDAuMTAlMjIlMkMlMEEpJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJnb29nbGUlMkZnZW1tYS0yYiUyMiUwQSUwQWNvbmZpZyUyMCUzRCUyMEF1dG9Db25maWcuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9jb25maWcoY29uZmlnKS50bygwKSUwQSUwQXRyYWluZXIlMjAlM0QlMjB0cmwuU0ZUVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTIwJTBBJTIwJTIwJTIwJTIwYXJncyUzRGFyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEdHJhaW5fZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMGRhdGFzZXRfdGV4dF9maWVsZCUzRCd0ZXh0JyUyQyUwQSUyMCUyMCUyMCUyMG1heF9zZXFfbGVuZ3RoJTNENTEyJTJDJTBBKSUwQSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> datasets
<span class="hljs-keyword">import</span> trl

<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, AutoConfig, AutoTokenizer, AutoModelForCausalLM

train_dataset = datasets.load_dataset(<span class="hljs-string">&#x27;imdb&#x27;</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>)

args = TrainingArguments(
    output_dir=<span class="hljs-string">&quot;./test-galore&quot;</span>,
    max_steps=<span class="hljs-number">100</span>,
    per_device_train_batch_size=<span class="hljs-number">2</span>,
    optim=<span class="hljs-string">&quot;galore_adamw&quot;</span>,
    optim_target_modules=[<span class="hljs-string">&quot;attn&quot;</span>, <span class="hljs-string">&quot;mlp&quot;</span>],
    optim_args=<span class="hljs-string">&quot;rank=64, update_proj_gap=100, scale=0.10&quot;</span>,
)

model_id = <span class="hljs-string">&quot;google/gemma-2b&quot;</span>

config = AutoConfig.from_pretrained(model_id)

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_config(config).to(<span class="hljs-number">0</span>)

trainer = trl.SFTTrainer(
    model=model, 
    args=args,
    train_dataset=train_dataset,
    dataset_text_field=<span class="hljs-string">&#x27;text&#x27;</span>,
    max_seq_length=<span class="hljs-number">512</span>,
)

trainer.train()`,wrap:!1}}),Ql=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjBkYXRhc2V0cyUwQWltcG9ydCUyMHRybCUwQSUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUcmFpbmluZ0FyZ3VtZW50cyUyQyUyMEF1dG9Db25maWclMkMlMjBBdXRvVG9rZW5pemVyJTJDJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEF0cmFpbl9kYXRhc2V0JTIwJTNEJTIwZGF0YXNldHMubG9hZF9kYXRhc2V0KCdpbWRiJyUyQyUyMHNwbGl0JTNEJ3RyYWluJyklMEElMEFhcmdzJTIwJTNEJTIwVHJhaW5pbmdBcmd1bWVudHMoJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMi4lMkZ0ZXN0LWdhbG9yZSUyMiUyQyUwQSUyMCUyMCUyMCUyMG1heF9zdGVwcyUzRDEwMCUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDIlMkMlMEElMjAlMjAlMjAlMjBvcHRpbSUzRCUyMmdhbG9yZV9hZGFtd19sYXllcndpc2UlMjIlMkMlMEElMjAlMjAlMjAlMjBvcHRpbV90YXJnZXRfbW9kdWxlcyUzRCU1QiUyMmF0dG4lMjIlMkMlMjAlMjJtbHAlMjIlNUQlMEEpJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJnb29nbGUlMkZnZW1tYS0yYiUyMiUwQSUwQWNvbmZpZyUyMCUzRCUyMEF1dG9Db25maWcuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9jb25maWcoY29uZmlnKS50bygwKSUwQSUwQXRyYWluZXIlMjAlM0QlMjB0cmwuU0ZUVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTIwJTBBJTIwJTIwJTIwJTIwYXJncyUzRGFyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEdHJhaW5fZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMGRhdGFzZXRfdGV4dF9maWVsZCUzRCd0ZXh0JyUyQyUwQSUyMCUyMCUyMCUyMG1heF9zZXFfbGVuZ3RoJTNENTEyJTJDJTBBKSUwQSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> datasets
<span class="hljs-keyword">import</span> trl

<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, AutoConfig, AutoTokenizer, AutoModelForCausalLM

train_dataset = datasets.load_dataset(<span class="hljs-string">&#x27;imdb&#x27;</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>)

args = TrainingArguments(
    output_dir=<span class="hljs-string">&quot;./test-galore&quot;</span>,
    max_steps=<span class="hljs-number">100</span>,
    per_device_train_batch_size=<span class="hljs-number">2</span>,
    optim=<span class="hljs-string">&quot;galore_adamw_layerwise&quot;</span>,
    optim_target_modules=[<span class="hljs-string">&quot;attn&quot;</span>, <span class="hljs-string">&quot;mlp&quot;</span>]
)

model_id = <span class="hljs-string">&quot;google/gemma-2b&quot;</span>

config = AutoConfig.from_pretrained(model_id)

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_config(config).to(<span class="hljs-number">0</span>)

trainer = trl.SFTTrainer(
    model=model, 
    args=args,
    train_dataset=train_dataset,
    dataset_text_field=<span class="hljs-string">&#x27;text&#x27;</span>,
    max_seq_length=<span class="hljs-number">512</span>,
)

trainer.train()`,wrap:!1}}),Fl=new Z({props:{title:"LOMO 옵티마이저",local:"lomo-optimizer",headingTag:"h2"}}),X=new Kl({props:{$$slots:{default:[Ct]},$$scope:{ctx:h}}}),Nl=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjBkYXRhc2V0cyUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUcmFpbmluZ0FyZ3VtZW50cyUyQyUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQWltcG9ydCUyMHRybCUwQSUwQXRyYWluX2RhdGFzZXQlMjAlM0QlMjBkYXRhc2V0cy5sb2FkX2RhdGFzZXQoJ2ltZGInJTJDJTIwc3BsaXQlM0QndHJhaW4nKSUwQSUwQWFyZ3MlMjAlM0QlMjBUcmFpbmluZ0FyZ3VtZW50cyglMEElMjAlMjAlMjAlMjBvdXRwdXRfZGlyJTNEJTIyLiUyRnRlc3QtbG9tbyUyMiUyQyUwQSUyMCUyMCUyMCUyMG1heF9zdGVwcyUzRDEwMDAlMkMlMEElMjAlMjAlMjAlMjBwZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlM0Q0JTJDJTBBJTIwJTIwJTIwJTIwb3B0aW0lM0QlMjJhZGFsb21vJTIyJTJDJTBBJTIwJTIwJTIwJTIwZ3JhZGllbnRfY2hlY2twb2ludGluZyUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBsb2dnaW5nX3N0cmF0ZWd5JTNEJTIyc3RlcHMlMjIlMkMlMEElMjAlMjAlMjAlMjBsb2dnaW5nX3N0ZXBzJTNEMSUyQyUwQSUyMCUyMCUyMCUyMGxlYXJuaW5nX3JhdGUlM0QyZS02JTJDJTBBJTIwJTIwJTIwJTIwc2F2ZV9zdHJhdGVneSUzRCUyMm5vJTIyJTJDJTBBJTIwJTIwJTIwJTIwcnVuX25hbWUlM0QlMjJsb21vLWltZGIlMjIlMkMlMEEpJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJnb29nbGUlMkZnZW1tYS0yYiUyMiUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwbG93X2NwdV9tZW1fdXNhZ2UlM0RUcnVlKS50bygwKSUwQSUwQXRyYWluZXIlMjAlM0QlMjB0cmwuU0ZUVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTIwJTBBJTIwJTIwJTIwJTIwYXJncyUzRGFyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEdHJhaW5fZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMGRhdGFzZXRfdGV4dF9maWVsZCUzRCd0ZXh0JyUyQyUwQSUyMCUyMCUyMCUyMG1heF9zZXFfbGVuZ3RoJTNEMTAyNCUyQyUwQSklMEElMEF0cmFpbmVyLnRyYWluKCk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> datasets
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, AutoTokenizer, AutoModelForCausalLM
<span class="hljs-keyword">import</span> trl

train_dataset = datasets.load_dataset(<span class="hljs-string">&#x27;imdb&#x27;</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>)

args = TrainingArguments(
    output_dir=<span class="hljs-string">&quot;./test-lomo&quot;</span>,
    max_steps=<span class="hljs-number">1000</span>,
    per_device_train_batch_size=<span class="hljs-number">4</span>,
    optim=<span class="hljs-string">&quot;adalomo&quot;</span>,
    gradient_checkpointing=<span class="hljs-literal">True</span>,
    logging_strategy=<span class="hljs-string">&quot;steps&quot;</span>,
    logging_steps=<span class="hljs-number">1</span>,
    learning_rate=<span class="hljs-number">2e-6</span>,
    save_strategy=<span class="hljs-string">&quot;no&quot;</span>,
    run_name=<span class="hljs-string">&quot;lomo-imdb&quot;</span>,
)

model_id = <span class="hljs-string">&quot;google/gemma-2b&quot;</span>

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(model_id, low_cpu_mem_usage=<span class="hljs-literal">True</span>).to(<span class="hljs-number">0</span>)

trainer = trl.SFTTrainer(
    model=model, 
    args=args,
    train_dataset=train_dataset,
    dataset_text_field=<span class="hljs-string">&#x27;text&#x27;</span>,
    max_seq_length=<span class="hljs-number">1024</span>,
)

trainer.train()`,wrap:!1}}),Gl=new Z({props:{title:"Accelerate와 Trainer",local:"accelerate-and-trainer",headingTag:"h2"}}),Q=new Kl({props:{$$slots:{default:[Zt]},$$scope:{ctx:h}}}),R=new mt({props:{id:"config",options:["DistributedDataParallel","FSDP","DeepSpeed","DeepSpeed with Accelerate plugin"],$$slots:{default:[Qt]},$$scope:{ctx:h}}}),Hl=new g({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMCU1QyUwQSUyMCUyMCUyMCUyMC4lMkZleGFtcGxlcyUyRnB5dG9yY2glMkZ0ZXh0LWNsYXNzaWZpY2F0aW9uJTJGcnVuX2dsdWUucHklMjAlNUMlMEElMjAlMjAlMjAlMjAtLW1vZGVsX25hbWVfb3JfcGF0aCUyMGdvb2dsZS1iZXJ0JTJGYmVydC1iYXNlLWNhc2VkJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS10YXNrX25hbWUlMjAlMjRUQVNLX05BTUUlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWRvX3RyYWluJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1kb19ldmFsJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tYXhfc2VxX2xlbmd0aCUyMDEyOCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIwMTYlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWxlYXJuaW5nX3JhdGUlMjA1ZS01JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1udW1fdHJhaW5fZXBvY2hzJTIwMyUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tb3V0cHV0X2RpciUyMCUyRnRtcCUyRiUyNFRBU0tfTkFNRSUyRiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tb3ZlcndyaXRlX291dHB1dF9kaXI=",highlighted:`accelerate launch \\
    ./examples/pytorch/text-classification/run_glue.py \\
    --model_name_or_path google-bert/bert-base-cased \\
    --task_name <span class="hljs-variable">$TASK_NAME</span> \\
    --do_train \\
    --do_eval \\
    --max_seq_length 128 \\
    --per_device_train_batch_size 16 \\
    --learning_rate 5e-5 \\
    --num_train_epochs 3 \\
    --output_dir /tmp/<span class="hljs-variable">$TASK_NAME</span>/ \\
    --overwrite_output_dir`,wrap:!1}}),xl=new g({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tbnVtX3Byb2Nlc3NlcyUzRDIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXVzZV9mc2RwJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1taXhlZF9wcmVjaXNpb24lM0RiZjE2JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1mc2RwX2F1dG9fd3JhcF9wb2xpY3klM0RUUkFOU0ZPUk1FUl9CQVNFRF9XUkFQJTIwJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1mc2RwX3RyYW5zZm9ybWVyX2xheWVyX2Nsc190b193cmFwJTNEJTIyQmVydExheWVyJTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1mc2RwX3NoYXJkaW5nX3N0cmF0ZWd5JTNEMSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tZnNkcF9zdGF0ZV9kaWN0X3R5cGUlM0RGVUxMX1NUQVRFX0RJQ1QlMjAlNUMlMEElMjAlMjAlMjAlMjAuJTJGZXhhbXBsZXMlMkZweXRvcmNoJTJGdGV4dC1jbGFzc2lmaWNhdGlvbiUyRnJ1bl9nbHVlLnB5JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tb2RlbF9uYW1lX29yX3BhdGglMjBnb29nbGUtYmVydCUyRmJlcnQtYmFzZS1jYXNlZCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tdGFza19uYW1lJTIwJTI0VEFTS19OQU1FJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1kb190cmFpbiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tZG9fZXZhbCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbWF4X3NlcV9sZW5ndGglMjAxMjglMjAlNUMlMEElMjAlMjAlMjAlMjAtLXBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUyMDE2JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1sZWFybmluZ19yYXRlJTIwNWUtNSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbnVtX3RyYWluX2Vwb2NocyUyMDMlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW91dHB1dF9kaXIlMjAlMkZ0bXAlMkYlMjRUQVNLX05BTUUlMkYlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW92ZXJ3cml0ZV9vdXRwdXRfZGly",highlighted:`accelerate launch --num_processes=2 \\
    --use_fsdp \\
    --mixed_precision=bf16 \\
    --fsdp_auto_wrap_policy=TRANSFORMER_BASED_WRAP  \\
    --fsdp_transformer_layer_cls_to_wrap=<span class="hljs-string">&quot;BertLayer&quot;</span> \\
    --fsdp_sharding_strategy=1 \\
    --fsdp_state_dict_type=FULL_STATE_DICT \\
    ./examples/pytorch/text-classification/run_glue.py \\
    --model_name_or_path google-bert/bert-base-cased \\
    --task_name <span class="hljs-variable">$TASK_NAME</span> \\
    --do_train \\
    --do_eval \\
    --max_seq_length 128 \\
    --per_device_train_batch_size 16 \\
    --learning_rate 5e-5 \\
    --num_train_epochs 3 \\
    --output_dir /tmp/<span class="hljs-variable">$TASK_NAME</span>/ \\
    --overwrite_output_dir`,wrap:!1}}),Dl=new ft({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/ko/trainer.md"}}),{c(){a=c("meta"),r=M(),n=c("p"),u=M(),m(o.$$.fragment),f=M(),b=c("p"),b.innerHTML=_,J=M(),m(I.$$.fragment),F=M(),C=c("p"),C.innerHTML=Pl,E=M(),m(W.$$.fragment),ls=M(),N=c("p"),N.innerHTML=de,ss=M(),m(G.$$.fragment),es=M(),k=c("p"),k.innerHTML=ue,ts=M(),v=c("ol"),v.innerHTML=fe,as=M(),S=c("p"),S.innerHTML=he,ns=M(),Y=c("p"),Y.innerHTML=be,Ms=M(),m(H.$$.fragment),ps=M(),z=c("p"),z.innerHTML=Ie,rs=M(),x=c("p"),x.innerHTML=_e,cs=M(),m(L.$$.fragment),is=M(),m(D.$$.fragment),Ts=M(),q=c("p"),q.innerHTML=ge,os=M(),m(P.$$.fragment),Js=M(),O=c("p"),O.innerHTML=Ae,ms=M(),K=c("ul"),K.innerHTML=Ce,js=M(),ll=c("p"),ll.innerHTML=Ze,ws=M(),m(sl.$$.fragment),ys=M(),el=c("p"),el.innerHTML=Ve,Us=M(),tl=c("ul"),tl.innerHTML=$e,ds=M(),al=c("p"),al.innerHTML=Be,us=M(),m(nl.$$.fragment),fs=M(),m(Ml.$$.fragment),hs=M(),pl=c("p"),pl.innerHTML=Xe,bs=M(),rl=c("p"),rl.textContent=Qe,Is=M(),m(cl.$$.fragment),_s=M(),il=c("p"),il.innerHTML=Re,gs=M(),m(Tl.$$.fragment),As=M(),m(ol.$$.fragment),Cs=M(),m(V.$$.fragment),Zs=M(),Jl=c("p"),Jl.innerHTML=Fe,Vs=M(),ml=c("p"),ml.innerHTML=Ee,$s=M(),m($.$$.fragment),Bs=M(),jl=c("p"),jl.textContent=We,Xs=M(),m(wl.$$.fragment),Qs=M(),yl=c("p"),yl.innerHTML=Ne,Rs=M(),m(B.$$.fragment),Fs=M(),m(Ul.$$.fragment),Es=M(),dl=c("p"),dl.innerHTML=Ge,Ws=M(),m(ul.$$.fragment),Ns=M(),fl=c("p"),fl.textContent=ke,Gs=M(),m(hl.$$.fragment),ks=M(),bl=c("p"),bl.textContent=ve,vs=M(),Il=c("p"),Il.textContent=Se,Ss=M(),m(_l.$$.fragment),Ys=M(),gl=c("p"),gl.innerHTML=Ye,Hs=M(),m(Al.$$.fragment),zs=M(),Cl=c("p"),Cl.innerHTML=He,xs=M(),m(Zl.$$.fragment),Ls=M(),Vl=c("p"),Vl.innerHTML=ze,Ds=M(),$l=c("p"),$l.textContent=xe,qs=M(),Bl=c("p"),Bl.textContent=Le,Ps=M(),Xl=c("p"),Xl.innerHTML=De,Os=M(),m(Ql.$$.fragment),Ks=M(),Rl=c("p"),Rl.innerHTML=qe,le=M(),m(Fl.$$.fragment),se=M(),El=c("p"),El.innerHTML=Pe,ee=M(),m(X.$$.fragment),te=M(),Wl=c("p"),Wl.innerHTML=Oe,ae=M(),m(Nl.$$.fragment),ne=M(),m(Gl.$$.fragment),Me=M(),kl=c("p"),kl.innerHTML=Ke,pe=M(),m(Q.$$.fragment),re=M(),vl=c("p"),vl.innerHTML=lt,ce=M(),m(R.$$.fragment),ie=M(),Sl=c("p"),Sl.innerHTML=st,Te=M(),Yl=c("p"),Yl.innerHTML=et,oe=M(),m(Hl.$$.fragment),Je=M(),zl=c("p"),zl.innerHTML=tt,me=M(),m(xl.$$.fragment),je=M(),Ll=c("p"),Ll.innerHTML=at,we=M(),m(Dl.$$.fragment),ye=M(),Ol=c("p"),this.h()},l(l){const s=dt("svelte-u9bgzb",document.head);a=i(s,"META",{name:!0,content:!0}),s.forEach(e),r=p(l),n=i(l,"P",{}),ot(n).forEach(e),u=p(l),j(o.$$.fragment,l),f=p(l),b=i(l,"P",{"data-svelte-h":!0}),T(b)!=="svelte-1r4aiss"&&(b.innerHTML=_),J=p(l),j(I.$$.fragment,l),F=p(l),C=i(l,"P",{"data-svelte-h":!0}),T(C)!=="svelte-1cp0146"&&(C.innerHTML=Pl),E=p(l),j(W.$$.fragment,l),ls=p(l),N=i(l,"P",{"data-svelte-h":!0}),T(N)!=="svelte-1eiyo2g"&&(N.innerHTML=de),ss=p(l),j(G.$$.fragment,l),es=p(l),k=i(l,"P",{"data-svelte-h":!0}),T(k)!=="svelte-1acln4k"&&(k.innerHTML=ue),ts=p(l),v=i(l,"OL",{"data-svelte-h":!0}),T(v)!=="svelte-1paahjl"&&(v.innerHTML=fe),as=p(l),S=i(l,"P",{"data-svelte-h":!0}),T(S)!=="svelte-3o0je1"&&(S.innerHTML=he),ns=p(l),Y=i(l,"P",{"data-svelte-h":!0}),T(Y)!=="svelte-4zmz0l"&&(Y.innerHTML=be),Ms=p(l),j(H.$$.fragment,l),ps=p(l),z=i(l,"P",{"data-svelte-h":!0}),T(z)!=="svelte-rthqdp"&&(z.innerHTML=Ie),rs=p(l),x=i(l,"P",{"data-svelte-h":!0}),T(x)!=="svelte-a31jzj"&&(x.innerHTML=_e),cs=p(l),j(L.$$.fragment,l),is=p(l),j(D.$$.fragment,l),Ts=p(l),q=i(l,"P",{"data-svelte-h":!0}),T(q)!=="svelte-b83gro"&&(q.innerHTML=ge),os=p(l),j(P.$$.fragment,l),Js=p(l),O=i(l,"P",{"data-svelte-h":!0}),T(O)!=="svelte-1qy235s"&&(O.innerHTML=Ae),ms=p(l),K=i(l,"UL",{"data-svelte-h":!0}),T(K)!=="svelte-1biqkkg"&&(K.innerHTML=Ce),js=p(l),ll=i(l,"P",{"data-svelte-h":!0}),T(ll)!=="svelte-iwkjnt"&&(ll.innerHTML=Ze),ws=p(l),j(sl.$$.fragment,l),ys=p(l),el=i(l,"P",{"data-svelte-h":!0}),T(el)!=="svelte-tbyxn3"&&(el.innerHTML=Ve),Us=p(l),tl=i(l,"UL",{"data-svelte-h":!0}),T(tl)!=="svelte-2gbdwc"&&(tl.innerHTML=$e),ds=p(l),al=i(l,"P",{"data-svelte-h":!0}),T(al)!=="svelte-vw04s9"&&(al.innerHTML=Be),us=p(l),j(nl.$$.fragment,l),fs=p(l),j(Ml.$$.fragment,l),hs=p(l),pl=i(l,"P",{"data-svelte-h":!0}),T(pl)!=="svelte-gudmov"&&(pl.innerHTML=Xe),bs=p(l),rl=i(l,"P",{"data-svelte-h":!0}),T(rl)!=="svelte-1sr1d63"&&(rl.textContent=Qe),Is=p(l),j(cl.$$.fragment,l),_s=p(l),il=i(l,"P",{"data-svelte-h":!0}),T(il)!=="svelte-c4o7g1"&&(il.innerHTML=Re),gs=p(l),j(Tl.$$.fragment,l),As=p(l),j(ol.$$.fragment,l),Cs=p(l),j(V.$$.fragment,l),Zs=p(l),Jl=i(l,"P",{"data-svelte-h":!0}),T(Jl)!=="svelte-1igbvzt"&&(Jl.innerHTML=Fe),Vs=p(l),ml=i(l,"P",{"data-svelte-h":!0}),T(ml)!=="svelte-1xaqk62"&&(ml.innerHTML=Ee),$s=p(l),j($.$$.fragment,l),Bs=p(l),jl=i(l,"P",{"data-svelte-h":!0}),T(jl)!=="svelte-1dt04b5"&&(jl.textContent=We),Xs=p(l),j(wl.$$.fragment,l),Qs=p(l),yl=i(l,"P",{"data-svelte-h":!0}),T(yl)!=="svelte-1dk6uay"&&(yl.innerHTML=Ne),Rs=p(l),j(B.$$.fragment,l),Fs=p(l),j(Ul.$$.fragment,l),Es=p(l),dl=i(l,"P",{"data-svelte-h":!0}),T(dl)!=="svelte-1hypqsm"&&(dl.innerHTML=Ge),Ws=p(l),j(ul.$$.fragment,l),Ns=p(l),fl=i(l,"P",{"data-svelte-h":!0}),T(fl)!=="svelte-1ft34jp"&&(fl.textContent=ke),Gs=p(l),j(hl.$$.fragment,l),ks=p(l),bl=i(l,"P",{"data-svelte-h":!0}),T(bl)!=="svelte-zig1fw"&&(bl.textContent=ve),vs=p(l),Il=i(l,"P",{"data-svelte-h":!0}),T(Il)!=="svelte-15aiu37"&&(Il.textContent=Se),Ss=p(l),j(_l.$$.fragment,l),Ys=p(l),gl=i(l,"P",{"data-svelte-h":!0}),T(gl)!=="svelte-13s90m9"&&(gl.innerHTML=Ye),Hs=p(l),j(Al.$$.fragment,l),zs=p(l),Cl=i(l,"P",{"data-svelte-h":!0}),T(Cl)!=="svelte-q0mtk0"&&(Cl.innerHTML=He),xs=p(l),j(Zl.$$.fragment,l),Ls=p(l),Vl=i(l,"P",{"data-svelte-h":!0}),T(Vl)!=="svelte-9wx27h"&&(Vl.innerHTML=ze),Ds=p(l),$l=i(l,"P",{"data-svelte-h":!0}),T($l)!=="svelte-3m2n0g"&&($l.textContent=xe),qs=p(l),Bl=i(l,"P",{"data-svelte-h":!0}),T(Bl)!=="svelte-o488v2"&&(Bl.textContent=Le),Ps=p(l),Xl=i(l,"P",{"data-svelte-h":!0}),T(Xl)!=="svelte-6sty29"&&(Xl.innerHTML=De),Os=p(l),j(Ql.$$.fragment,l),Ks=p(l),Rl=i(l,"P",{"data-svelte-h":!0}),T(Rl)!=="svelte-1h6bbfe"&&(Rl.innerHTML=qe),le=p(l),j(Fl.$$.fragment,l),se=p(l),El=i(l,"P",{"data-svelte-h":!0}),T(El)!=="svelte-19j518g"&&(El.innerHTML=Pe),ee=p(l),j(X.$$.fragment,l),te=p(l),Wl=i(l,"P",{"data-svelte-h":!0}),T(Wl)!=="svelte-16ffc3o"&&(Wl.innerHTML=Oe),ae=p(l),j(Nl.$$.fragment,l),ne=p(l),j(Gl.$$.fragment,l),Me=p(l),kl=i(l,"P",{"data-svelte-h":!0}),T(kl)!=="svelte-145u0tn"&&(kl.innerHTML=Ke),pe=p(l),j(Q.$$.fragment,l),re=p(l),vl=i(l,"P",{"data-svelte-h":!0}),T(vl)!=="svelte-1j99ily"&&(vl.innerHTML=lt),ce=p(l),j(R.$$.fragment,l),ie=p(l),Sl=i(l,"P",{"data-svelte-h":!0}),T(Sl)!=="svelte-y960yk"&&(Sl.innerHTML=st),Te=p(l),Yl=i(l,"P",{"data-svelte-h":!0}),T(Yl)!=="svelte-uvyyka"&&(Yl.innerHTML=et),oe=p(l),j(Hl.$$.fragment,l),Je=p(l),zl=i(l,"P",{"data-svelte-h":!0}),T(zl)!=="svelte-1nohodh"&&(zl.innerHTML=tt),me=p(l),j(xl.$$.fragment,l),je=p(l),Ll=i(l,"P",{"data-svelte-h":!0}),T(Ll)!=="svelte-1xu5fm6"&&(Ll.innerHTML=at),we=p(l),j(Dl.$$.fragment,l),ye=p(l),Ol=i(l,"P",{}),ot(Ol).forEach(e),this.h()},h(){Jt(a,"name","hf:doc:metadata"),Jt(a,"content",Ft)},m(l,s){ut(document.head,a),t(l,r,s),t(l,n,s),t(l,u,s),w(o,l,s),t(l,f,s),t(l,b,s),t(l,J,s),w(I,l,s),t(l,F,s),t(l,C,s),t(l,E,s),w(W,l,s),t(l,ls,s),t(l,N,s),t(l,ss,s),w(G,l,s),t(l,es,s),t(l,k,s),t(l,ts,s),t(l,v,s),t(l,as,s),t(l,S,s),t(l,ns,s),t(l,Y,s),t(l,Ms,s),w(H,l,s),t(l,ps,s),t(l,z,s),t(l,rs,s),t(l,x,s),t(l,cs,s),w(L,l,s),t(l,is,s),w(D,l,s),t(l,Ts,s),t(l,q,s),t(l,os,s),w(P,l,s),t(l,Js,s),t(l,O,s),t(l,ms,s),t(l,K,s),t(l,js,s),t(l,ll,s),t(l,ws,s),w(sl,l,s),t(l,ys,s),t(l,el,s),t(l,Us,s),t(l,tl,s),t(l,ds,s),t(l,al,s),t(l,us,s),w(nl,l,s),t(l,fs,s),w(Ml,l,s),t(l,hs,s),t(l,pl,s),t(l,bs,s),t(l,rl,s),t(l,Is,s),w(cl,l,s),t(l,_s,s),t(l,il,s),t(l,gs,s),w(Tl,l,s),t(l,As,s),w(ol,l,s),t(l,Cs,s),w(V,l,s),t(l,Zs,s),t(l,Jl,s),t(l,Vs,s),t(l,ml,s),t(l,$s,s),w($,l,s),t(l,Bs,s),t(l,jl,s),t(l,Xs,s),w(wl,l,s),t(l,Qs,s),t(l,yl,s),t(l,Rs,s),w(B,l,s),t(l,Fs,s),w(Ul,l,s),t(l,Es,s),t(l,dl,s),t(l,Ws,s),w(ul,l,s),t(l,Ns,s),t(l,fl,s),t(l,Gs,s),w(hl,l,s),t(l,ks,s),t(l,bl,s),t(l,vs,s),t(l,Il,s),t(l,Ss,s),w(_l,l,s),t(l,Ys,s),t(l,gl,s),t(l,Hs,s),w(Al,l,s),t(l,zs,s),t(l,Cl,s),t(l,xs,s),w(Zl,l,s),t(l,Ls,s),t(l,Vl,s),t(l,Ds,s),t(l,$l,s),t(l,qs,s),t(l,Bl,s),t(l,Ps,s),t(l,Xl,s),t(l,Os,s),w(Ql,l,s),t(l,Ks,s),t(l,Rl,s),t(l,le,s),w(Fl,l,s),t(l,se,s),t(l,El,s),t(l,ee,s),w(X,l,s),t(l,te,s),t(l,Wl,s),t(l,ae,s),w(Nl,l,s),t(l,ne,s),w(Gl,l,s),t(l,Me,s),t(l,kl,s),t(l,pe,s),w(Q,l,s),t(l,re,s),t(l,vl,s),t(l,ce,s),w(R,l,s),t(l,ie,s),t(l,Sl,s),t(l,Te,s),t(l,Yl,s),t(l,oe,s),w(Hl,l,s),t(l,Je,s),t(l,zl,s),t(l,me,s),w(xl,l,s),t(l,je,s),t(l,Ll,s),t(l,we,s),w(Dl,l,s),t(l,ye,s),t(l,Ol,s),Ue=!0},p(l,[s]){const nt={};s&2&&(nt.$$scope={dirty:s,ctx:l}),I.$set(nt);const Mt={};s&2&&(Mt.$$scope={dirty:s,ctx:l}),V.$set(Mt);const pt={};s&2&&(pt.$$scope={dirty:s,ctx:l}),$.$set(pt);const rt={};s&2&&(rt.$$scope={dirty:s,ctx:l}),B.$set(rt);const ct={};s&2&&(ct.$$scope={dirty:s,ctx:l}),X.$set(ct);const it={};s&2&&(it.$$scope={dirty:s,ctx:l}),Q.$set(it);const Tt={};s&2&&(Tt.$$scope={dirty:s,ctx:l}),R.$set(Tt)},i(l){Ue||(y(o.$$.fragment,l),y(I.$$.fragment,l),y(W.$$.fragment,l),y(G.$$.fragment,l),y(H.$$.fragment,l),y(L.$$.fragment,l),y(D.$$.fragment,l),y(P.$$.fragment,l),y(sl.$$.fragment,l),y(nl.$$.fragment,l),y(Ml.$$.fragment,l),y(cl.$$.fragment,l),y(Tl.$$.fragment,l),y(ol.$$.fragment,l),y(V.$$.fragment,l),y($.$$.fragment,l),y(wl.$$.fragment,l),y(B.$$.fragment,l),y(Ul.$$.fragment,l),y(ul.$$.fragment,l),y(hl.$$.fragment,l),y(_l.$$.fragment,l),y(Al.$$.fragment,l),y(Zl.$$.fragment,l),y(Ql.$$.fragment,l),y(Fl.$$.fragment,l),y(X.$$.fragment,l),y(Nl.$$.fragment,l),y(Gl.$$.fragment,l),y(Q.$$.fragment,l),y(R.$$.fragment,l),y(Hl.$$.fragment,l),y(xl.$$.fragment,l),y(Dl.$$.fragment,l),Ue=!0)},o(l){U(o.$$.fragment,l),U(I.$$.fragment,l),U(W.$$.fragment,l),U(G.$$.fragment,l),U(H.$$.fragment,l),U(L.$$.fragment,l),U(D.$$.fragment,l),U(P.$$.fragment,l),U(sl.$$.fragment,l),U(nl.$$.fragment,l),U(Ml.$$.fragment,l),U(cl.$$.fragment,l),U(Tl.$$.fragment,l),U(ol.$$.fragment,l),U(V.$$.fragment,l),U($.$$.fragment,l),U(wl.$$.fragment,l),U(B.$$.fragment,l),U(Ul.$$.fragment,l),U(ul.$$.fragment,l),U(hl.$$.fragment,l),U(_l.$$.fragment,l),U(Al.$$.fragment,l),U(Zl.$$.fragment,l),U(Ql.$$.fragment,l),U(Fl.$$.fragment,l),U(X.$$.fragment,l),U(Nl.$$.fragment,l),U(Gl.$$.fragment,l),U(Q.$$.fragment,l),U(R.$$.fragment,l),U(Hl.$$.fragment,l),U(xl.$$.fragment,l),U(Dl.$$.fragment,l),Ue=!1},d(l){l&&(e(r),e(n),e(u),e(f),e(b),e(J),e(F),e(C),e(E),e(ls),e(N),e(ss),e(es),e(k),e(ts),e(v),e(as),e(S),e(ns),e(Y),e(Ms),e(ps),e(z),e(rs),e(x),e(cs),e(is),e(Ts),e(q),e(os),e(Js),e(O),e(ms),e(K),e(js),e(ll),e(ws),e(ys),e(el),e(Us),e(tl),e(ds),e(al),e(us),e(fs),e(hs),e(pl),e(bs),e(rl),e(Is),e(_s),e(il),e(gs),e(As),e(Cs),e(Zs),e(Jl),e(Vs),e(ml),e($s),e(Bs),e(jl),e(Xs),e(Qs),e(yl),e(Rs),e(Fs),e(Es),e(dl),e(Ws),e(Ns),e(fl),e(Gs),e(ks),e(bl),e(vs),e(Il),e(Ss),e(Ys),e(gl),e(Hs),e(zs),e(Cl),e(xs),e(Ls),e(Vl),e(Ds),e($l),e(qs),e(Bl),e(Ps),e(Xl),e(Os),e(Ks),e(Rl),e(le),e(se),e(El),e(ee),e(te),e(Wl),e(ae),e(ne),e(Me),e(kl),e(pe),e(re),e(vl),e(ce),e(ie),e(Sl),e(Te),e(Yl),e(oe),e(Je),e(zl),e(me),e(je),e(Ll),e(we),e(ye),e(Ol)),e(a),d(o,l),d(I,l),d(W,l),d(G,l),d(H,l),d(L,l),d(D,l),d(P,l),d(sl,l),d(nl,l),d(Ml,l),d(cl,l),d(Tl,l),d(ol,l),d(V,l),d($,l),d(wl,l),d(B,l),d(Ul,l),d(ul,l),d(hl,l),d(_l,l),d(Al,l),d(Zl,l),d(Ql,l),d(Fl,l),d(X,l),d(Nl,l),d(Gl,l),d(Q,l),d(R,l),d(Hl,l),d(xl,l),d(Dl,l)}}}const Ft='{"title":"Trainer","local":"trainer","sections":[{"title":"기본 사용법","local":"basic-usage","sections":[{"title":"체크포인트","local":"checkpoints","sections":[],"depth":3}],"depth":2},{"title":"Trainer 맞춤 설정","local":"customize-the-trainer","sections":[{"title":"콜백","local":"callbacks","sections":[],"depth":3}],"depth":2},{"title":"로깅","local":"logging","sections":[],"depth":2},{"title":"NEFTune","local":"neftune","sections":[],"depth":2},{"title":"GaLore","local":"galore","sections":[],"depth":2},{"title":"LOMO 옵티마이저","local":"lomo-optimizer","sections":[],"depth":2},{"title":"Accelerate와 Trainer","local":"accelerate-and-trainer","sections":[],"depth":2}],"depth":1}';function Et(h){return wt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Yt extends yt{constructor(a){super(),Ut(this,a,Et,Rt,jt,{})}}export{Yt as component};
