import{s as Os,n as le,o as Me}from"../chunks/scheduler.39a43cde.js";import{S as se,i as ee,g as n,s as t,r as J,A as te,h as y,f as s,c as a,j as Ps,u as T,x as U,k as Ks,y as ae,a as e,v as j,d as p,t as w,w as I}from"../chunks/index.23ec2204.js";import{D as ne,C}from"../chunks/DocNotebookDropdown.21a75b74.js";import{H as Vl,E as ye}from"../chunks/EditOnGithub.f2d09226.js";function Ue(as){let i,Wl,Ql,Sl,m,Nl,o,gl,c,ns='<em>作者: <a href="https://github.com/MKhalusova" rel="nofollow">Maria Khalusova</a></em>',Yl,d,ys="公开发布的代码 LLM，如 Codex、StarCoder 和 Code Llama，在生成遵循通用编程原则和语法的代码方面表现出色，但它们可能不符合组织的内部惯例，或者不了解某些特定的库。",Hl,A,Us="在这个 notebook 中，我们将展示如何微调代码 LLM 来更好的理解你们公司或组织的代码风格和习惯。由于代码 LLM 非常大，按照传统的微调方式可能会消耗大量资源。但不用担心！我们会教你一些技巧，让你只用单个 GPU 就能完成微调工作。",vl,r,xl,b,Js='对于这个例子，我们选择了 GitHub 上 Hugging Face 的前 10 个公共仓库。我们已经排除了非代码文件，如图片、音频文件、演示文稿等。对于 Jupyter notebook，我们只保留了包含代码的单元格。生成的代码被存储为一个数据集，你可以在 Hugging Face Hub 上找到，位于 <a href="https://huggingface.co/datasets/smangrul/hf-stack-v1" rel="nofollow"><code>smangrul/hf-stack-v1</code></a>。它包含仓库 id、文件路径和文件内容。',zl,f,$l,u,Ts='我们将微调 <a href="https://huggingface.co/bigcode/starcoderbase-1b" rel="nofollow"><code>bigcode/starcoderbase-1b</code></a> 模型，这是一个在 80 多种编程语言上训练的 10 亿参数模型。这是一个需要权限的模型，所以如果你计划使用这个确切模型运行这个 notebook，你需要在其模型页面上获得访问权限。登录你的 Hugging Face 帐户以执行此操作：',Ll,h,Dl,Z,js="To get started, let’s install all the necessary libraries. As you can see, in addition to <code>transformers</code> and <code>datasets</code>, we’ll be using <code>peft</code>, <code>bitsandbytes</code>, and <code>flash-attn</code> to optimize the training.",ql,B,ps="By employing parameter-efficient training techniques, we can run this notebook on a single A100 High-RAM GPU.",Pl,_,Kl,G,ws="现在让我们定义一些变量。请随意调整这些变量。",Ol,R,lM,E,MM,k,sM,X,Is="首先加载数据。由于数据集可能相当大，请确保启用流模式。流模式允许我们在遍历数据集时逐步加载数据，而不是一次性下载数据集的整个内容。",eM,V,Cs="我们将前 4000 个示例作为验证集，其余的全部作为训练数据。",tM,Q,aM,F,is="在这一步，数据集仍然包含任意长度的原始数据。为了训练，我们需要固定长度的输入。让我们创建一个可迭代的数据集，它可以从文本文件流中返回固定长度的 token 块。",nM,W,ms="首先，让我们估计数据集中每个 token 的平均字符数，这将帮助我们稍后估计文本缓冲区中的 token 数量。默认情况下，我们只从数据集中取 400 个示例（<code>nb_examples</code>）。只使用整个数据集的一个子集可以减少计算成本，同时仍然提供了对整体字符到 token 比的合理估计。",yM,S,UM,N,os=`The character to token ratio of the dataset is: 2.43
`,JM,g,cs="字符到 token 的比也可以用作文本标记质量的一个指标。例如，字符到 token 的比为 1.0 意味着每个字符都由一个 token 表示，这并没有太多意义。表明标记化做得不好。在标准的英文文本中，一个 token 通常相当于大约四个字符，这意味着字符到 token 的比率大约是 4.0。我们可以预见在代码数据集中的比率会更低，但一般来说，2.0 到 3.5 之间的数字可以认为是足够好的。",TM,Y,ds=`<strong>可选的 FIM 变换</strong>
自回归语言模型通常是从左到右生成序列的。通过应用 FIM 变换，模型也可以学习填充文本。详细信息可以看<a href="https://arxiv.org/pdf/2207.14255.pdf" rel="nofollow">“Efficient Training of Language Models to Fill in the Middle” 这篇论文</a>了解这种技术。`,jM,H,As="我们将在下面定义 FIM 变换，并在创建可迭代数据集时使用它们。然而，如果你想省略变换步骤，请将 <code>fim_rate</code> 设置为 0。",pM,v,wM,x,rs="让我们定义 <code>ConstantLengthDataset</code>，这是一个可迭代的数据集，它将返回固定长度的 token 块。为此，我们将从原始数据集中读取文本缓冲区，直到达到大小限制，然后应用分词器将原始文本转换为 token 后的输入。可选项，我们可以在一些序列上执行 FIM 变换（受影响的序列比例由 <code>fim_rate</code> 控制）。",IM,z,bs="定义好后，我们可以从训练和验证数据中创建 <code>ConstantLengthDataset</code> 的实例。",CM,$,iM,L,mM,D,fs="现在数据已经准备好了，是时候加载模型了！我们将加载量化的模型。",oM,q,us="因为量化使用更少的位来表示数据，所以会减少内存使用。我们将使用 <code>bitsandbytes</code> 库来量化模型，因为它与 <code>transformers</code> 有很好的集成。我们需要做的只是定义一个 <code>bitsandbytes</code> 配置，然后在加载模型时使用它。",cM,P,hs="4 比特位量化有不同的变体，但通常我们推荐使用 NF4 量化以获得更好的性能（<code>bnb_4bit_quant_type=&quot;nf4&quot;</code>）。",dM,K,Zs="<code>bnb_4bit_use_double_quant</code> 选项在第一次量化后添加第二次量化，以节省每个参数额外的 0.4 位。",AM,O,Bs='要了解更多关于量化的信息，请查看 <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">“利用 bitsandbytes、4 比特位量化和 QLoRA 让 LLMs 更易于访问” 的博客</a>。',rM,ll,_s="定义好后，将配置传递给 <code>from_pretrained</code> 方法以加载量化的模型。",bM,Ml,fM,sl,Gs="当使用量化模型进行训练时，你需要调用 <code>prepare_model_for_kbit_training()</code> 函数来预处理量化模型以进行训练。",uM,el,hM,tl,Rs="现在量化模型已经准备好了，我们可以设置一个 LoRA 配置。LoRA 通过大幅减少可训练参数的数量，使得微调更加高效。",ZM,al,Es="要使用 LoRA 技术训练模型，我们需要将基础模型包装为 <code>PeftModel</code>。这涉及到使用 <code>LoraConfig</code> 定义 LoRA 配置，并使用 <code>get_peft_model()</code> 和 <code>LoraConfig</code> 包装原始模型。",BM,nl,ks='要了解更多关于 LoRA 及其参数的信息，请参考 <a href="https://huggingface.co/docs/peft/conceptual_guides/lora" rel="nofollow">PEFT 文档</a>。',_M,yl,GM,Ul,Xs=`trainable params: 5,554,176 || all params: 1,142,761,472 || trainable%: 0.4860310866343243
`,RM,Jl,Vs="可以看到，通过应用 LoRA 技术，我们现在只需要训练不到 1% 的参数。",EM,Tl,kM,jl,Qs="现在我们已经准备好了数据，并且优化了模型，我们可以将所有东西整合在一起开始训练。",XM,pl,Fs="要实例化一个 <code>Trainer</code>，你需要定义训练配置。最重要的是 <code>TrainingArguments</code>，这是一个包含所有用于配置训练的属性的类。",VM,wl,Ws="这些与你可能运行的任何其他类型的模型训练相似，所以我们这里不会详细说明。",QM,Il,FM,Cl,Ss="最后一步，实例化 <code>Trainer</code> 并调用 <code>train</code> 方法。",WM,il,SM,ml,Ns=`Training...
`,NM,ol,gs="最后，你可以将微调好的模型推送到你的 Hub 仓库中，并分享给你的团队。",gM,cl,YM,dl,HM,Al,Ys="一旦模型被上传到 Hub，我们就可以使用它进行推理。为此，我们首先初始化原始的基础模型及其分词器。接下来，我们需要将微调后的权重与基础模型合并。",vM,rl,xM,bl,Hs="现在我们可以使用合并后的模型进行推理。为了方便起见，我们将定义一个 <code>get_code_completion</code> 函数 - 请随意尝试文本生成参数！",zM,fl,$M,ul,vs="现在，为了获得代码补全，我们只需要调用 <code>get_code_complete</code> 函数，并将我们希望补全的前几行作为前缀传递，以及一个空字符串作为后缀。",LM,hl,DM,Zl,xs=`from peft import LoraConfig, TaskType, get_peft_model
from transformers import AutoModelForCausalLM
peft_config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    r=8,
    lora_alpha=32,
    target_modules=["q_proj", "v_proj"],
    lora_dropout=0.1,
    bias="none",
    modules_to_save=["q_proj", "v_proj"],
    inference_mode=False,
)
model = AutoModelForCausalLM.from_pretrained("gpt2")
model = get_peft_model(model, peft_config)
model.print_trainable_parameters()
`,qM,Bl,zs="作为刚刚在这个 notebook 中使用过 PEFT 库的人，你可以看到创建为 <code>LoraConfig</code> 函数的生成结果相当不错！",PM,_l,$s="如果你回到我们为推理实例化模型的单元格，并注释掉我们合并微调权重的行，你可以看到原始模型对于完全相同的前缀会生成什么内容：",KM,Gl,OM,Rl,Ls=`from peft import LoraConfig, TaskType, get_peft_model
from transformers import AutoModelForCausalLM
peft_config = LoraConfig(
    model_name_or_path="facebook/wav2vec2-base-960h",
    num_labels=1,
    num_features=1,
    num_hidden_layers=1,
    num_attention_heads=1,
    num_hidden_layers_per_attention_head=1,
    num_attention_heads_per_hidden_layer=1,
    hidden_size=1024,
    hidden_dropout_prob=0.1,
    hidden_act="gelu",
    hidden_act_dropout_prob=0.1,
    hidden
`,ls,El,Ds="尽管这是 Python 语法，但你可以看到原始模型并不理解 <code>LoraConfig</code> 应该做什么。",Ms,kl,qs='要了解这种高效参数微调与完全微调的比较，以及如何通过推理端点在 VS Code 中使用这样的模型作为你的编程助手(copilot)，或者在本地使用，请查看<a href="https://huggingface.co/blog/personal-copilot" rel="nofollow">“个人编程助手(copilot)：训练你自己的编码助手”博客</a>。这个 notebook 补充了原始博客内容。',ss,Xl,es,Fl,ts;return m=new ne({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/cookbook/blob/main/notebooks/zh-CN/fine_tuning_code_llm_on_single_gpu.ipynb"}]}}),o=new Vl({props:{title:"在单个 GPU 上针对自定义代码微调代码 LLM",local:"在单个-gpu-上针对自定义代码微调代码-llm",headingTag:"h1"}}),r=new Vl({props:{title:"数据集",local:"数据集",headingTag:"h2"}}),f=new Vl({props:{title:"模型",local:"模型",headingTag:"h2"}}),h=new C({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`,wrap:!1}}),_=new C({props:{code:"IXBpcCUyMGluc3RhbGwlMjAtcSUyMHRyYW5zZm9ybWVycyUyMGRhdGFzZXRzJTIwcGVmdCUyMGJpdHNhbmRieXRlcyUyMGZsYXNoLWF0dG4=",highlighted:"!pip install -q transformers datasets peft bitsandbytes flash-attn",wrap:!1}}),R=new C({props:{code:"TU9ERUwlMjAlM0QlMjAlMjJiaWdjb2RlJTJGc3RhcmNvZGVyYmFzZS0xYiUyMiUyMCUyMCUyMyUyME1vZGVsJTIwY2hlY2twb2ludCUyMG9uJTIwdGhlJTIwSHVnZ2luZyUyMEZhY2UlMjBIdWIlMEFEQVRBU0VUJTIwJTNEJTIwJTIyc21hbmdydWwlMkZoZi1zdGFjay12MSUyMiUyMCUyMCUyMyUyMERhdGFzZXQlMjBvbiUyMHRoZSUyMEh1Z2dpbmclMjBGYWNlJTIwSHViJTBBREFUQV9DT0xVTU4lMjAlM0QlMjAlMjJjb250ZW50JTIyJTIwJTIwJTIzJTIwQ29sdW1uJTIwbmFtZSUyMGNvbnRhaW5pbmclMjB0aGUlMjBjb2RlJTIwY29udGVudCUwQSUwQVNFUV9MRU5HVEglMjAlM0QlMjAyMDQ4JTIwJTIwJTIzJTIwU2VxdWVuY2UlMjBsZW5ndGglMEElMEElMjMlMjBUcmFpbmluZyUyMGFyZ3VtZW50cyUwQU1BWF9TVEVQUyUyMCUzRCUyMDIwMDAlMjAlMjAlMjMlMjBtYXhfc3RlcHMlMEFCQVRDSF9TSVpFJTIwJTNEJTIwMTYlMjAlMjAlMjMlMjBiYXRjaF9zaXplJTBBR1JfQUNDX1NURVBTJTIwJTNEJTIwMSUyMCUyMCUyMyUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUwQUxSJTIwJTNEJTIwNWUtNCUyMCUyMCUyMyUyMGxlYXJuaW5nX3JhdGUlMEFMUl9TQ0hFRFVMRVJfVFlQRSUyMCUzRCUyMCUyMmNvc2luZSUyMiUyMCUyMCUyMyUyMGxyX3NjaGVkdWxlcl90eXBlJTBBV0VJR0hUX0RFQ0FZJTIwJTNEJTIwMC4wMSUyMCUyMCUyMyUyMHdlaWdodF9kZWNheSUwQU5VTV9XQVJNVVBfU1RFUFMlMjAlM0QlMjAzMCUyMCUyMCUyMyUyMG51bV93YXJtdXBfc3RlcHMlMEFFVkFMX0ZSRVElMjAlM0QlMjAxMDAlMjAlMjAlMjMlMjBldmFsX2ZyZXElMEFTQVZFX0ZSRVElMjAlM0QlMjAxMDAlMjAlMjAlMjMlMjBzYXZlX2ZyZXElMEFMT0dfRlJFUSUyMCUzRCUyMDI1JTIwJTIwJTIzJTIwbG9nX2ZyZXElMEFPVVRQVVRfRElSJTIwJTNEJTIwJTIycGVmdC1zdGFyY29kZXItbG9yYS1hMTAwJTIyJTIwJTIwJTIzJTIwb3V0cHV0X2RpciUwQUJGMTYlMjAlM0QlMjBUcnVlJTIwJTIwJTIzJTIwYmYxNiUwQUZQMTYlMjAlM0QlMjBGYWxzZSUyMCUyMCUyMyUyMG5vX2ZwMTYlMEElMEElMjMlMjBGSU0lMjB0cmFzZm9ybWF0aW9ucyUyMGFyZ3VtZW50cyUwQUZJTV9SQVRFJTIwJTNEJTIwMC41JTIwJTIwJTIzJTIwZmltX3JhdGUlMEFGSU1fU1BNX1JBVEUlMjAlM0QlMjAwLjUlMjAlMjAlMjMlMjBmaW1fc3BtX3JhdGUlMEElMEElMjMlMjBMT1JBJTBBTE9SQV9SJTIwJTNEJTIwOCUyMCUyMCUyMyUyMGxvcmFfciUwQUxPUkFfQUxQSEElMjAlM0QlMjAzMiUyMCUyMCUyMyUyMGxvcmFfYWxwaGElMEFMT1JBX0RST1BPVVQlMjAlM0QlMjAwLjAlMjAlMjAlMjMlMjBsb3JhX2Ryb3BvdXQlMEFMT1JBX1RBUkdFVF9NT0RVTEVTJTIwJTNEJTIwJTIyY19wcm9qJTJDY19hdHRuJTJDcV9hdHRuJTJDY19mYyUyQ2NfcHJvaiUyMiUyMCUyMCUyMyUyMGxvcmFfdGFyZ2V0X21vZHVsZXMlMEElMEElMjMlMjBiaXRzYW5kYnl0ZXMlMjBjb25maWclMEFVU0VfTkVTVEVEX1FVQU5UJTIwJTNEJTIwVHJ1ZSUyMCUyMCUyMyUyMHVzZV9uZXN0ZWRfcXVhbnQlMEFCTkJfNEJJVF9DT01QVVRFX0RUWVBFJTIwJTNEJTIwJTIyYmZsb2F0MTYlMjIlMjAlMjAlMjMlMjBibmJfNGJpdF9jb21wdXRlX2R0eXBlJTBBJTBBU0VFRCUyMCUzRCUyMDA=",highlighted:`MODEL = <span class="hljs-string">&quot;bigcode/starcoderbase-1b&quot;</span>  <span class="hljs-comment"># Model checkpoint on the Hugging Face Hub</span>
DATASET = <span class="hljs-string">&quot;smangrul/hf-stack-v1&quot;</span>  <span class="hljs-comment"># Dataset on the Hugging Face Hub</span>
DATA_COLUMN = <span class="hljs-string">&quot;content&quot;</span>  <span class="hljs-comment"># Column name containing the code content</span>

SEQ_LENGTH = <span class="hljs-number">2048</span>  <span class="hljs-comment"># Sequence length</span>

<span class="hljs-comment"># Training arguments</span>
MAX_STEPS = <span class="hljs-number">2000</span>  <span class="hljs-comment"># max_steps</span>
BATCH_SIZE = <span class="hljs-number">16</span>  <span class="hljs-comment"># batch_size</span>
GR_ACC_STEPS = <span class="hljs-number">1</span>  <span class="hljs-comment"># gradient_accumulation_steps</span>
LR = <span class="hljs-number">5e-4</span>  <span class="hljs-comment"># learning_rate</span>
LR_SCHEDULER_TYPE = <span class="hljs-string">&quot;cosine&quot;</span>  <span class="hljs-comment"># lr_scheduler_type</span>
WEIGHT_DECAY = <span class="hljs-number">0.01</span>  <span class="hljs-comment"># weight_decay</span>
NUM_WARMUP_STEPS = <span class="hljs-number">30</span>  <span class="hljs-comment"># num_warmup_steps</span>
EVAL_FREQ = <span class="hljs-number">100</span>  <span class="hljs-comment"># eval_freq</span>
SAVE_FREQ = <span class="hljs-number">100</span>  <span class="hljs-comment"># save_freq</span>
LOG_FREQ = <span class="hljs-number">25</span>  <span class="hljs-comment"># log_freq</span>
OUTPUT_DIR = <span class="hljs-string">&quot;peft-starcoder-lora-a100&quot;</span>  <span class="hljs-comment"># output_dir</span>
BF16 = <span class="hljs-literal">True</span>  <span class="hljs-comment"># bf16</span>
FP16 = <span class="hljs-literal">False</span>  <span class="hljs-comment"># no_fp16</span>

<span class="hljs-comment"># FIM trasformations arguments</span>
FIM_RATE = <span class="hljs-number">0.5</span>  <span class="hljs-comment"># fim_rate</span>
FIM_SPM_RATE = <span class="hljs-number">0.5</span>  <span class="hljs-comment"># fim_spm_rate</span>

<span class="hljs-comment"># LORA</span>
LORA_R = <span class="hljs-number">8</span>  <span class="hljs-comment"># lora_r</span>
LORA_ALPHA = <span class="hljs-number">32</span>  <span class="hljs-comment"># lora_alpha</span>
LORA_DROPOUT = <span class="hljs-number">0.0</span>  <span class="hljs-comment"># lora_dropout</span>
LORA_TARGET_MODULES = <span class="hljs-string">&quot;c_proj,c_attn,q_attn,c_fc,c_proj&quot;</span>  <span class="hljs-comment"># lora_target_modules</span>

<span class="hljs-comment"># bitsandbytes config</span>
USE_NESTED_QUANT = <span class="hljs-literal">True</span>  <span class="hljs-comment"># use_nested_quant</span>
BNB_4BIT_COMPUTE_DTYPE = <span class="hljs-string">&quot;bfloat16&quot;</span>  <span class="hljs-comment"># bnb_4bit_compute_dtype</span>

SEED = <span class="hljs-number">0</span>`,wrap:!1}}),E=new C({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMCglMEElMjAlMjAlMjAlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUwQSUyMCUyMCUyMCUyMEF1dG9Ub2tlbml6ZXIlMkMlMEElMjAlMjAlMjAlMjBUcmFpbmVyJTJDJTBBJTIwJTIwJTIwJTIwVHJhaW5pbmdBcmd1bWVudHMlMkMlMEElMjAlMjAlMjAlMjBsb2dnaW5nJTJDJTBBJTIwJTIwJTIwJTIwc2V0X3NlZWQlMkMlMEElMjAlMjAlMjAlMjBCaXRzQW5kQnl0ZXNDb25maWclMkMlMEEpJTBBJTBBc2V0X3NlZWQoU0VFRCk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> (
    AutoModelForCausalLM,
    AutoTokenizer,
    Trainer,
    TrainingArguments,
    logging,
    set_seed,
    BitsAndBytesConfig,
)

set_seed(SEED)`,wrap:!1}}),k=new Vl({props:{title:"准备数据",local:"准备数据",headingTag:"h2"}}),Q=new C({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHFkbSUyMGltcG9ydCUyMHRxZG0lMEElMEElMEFkYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMERBVEFTRVQlMkMlMEElMjAlMjAlMjAlMjBkYXRhX2RpciUzRCUyMmRhdGElMjIlMkMlMEElMjAlMjAlMjAlMjBzcGxpdCUzRCUyMnRyYWluJTIyJTJDJTBBJTIwJTIwJTIwJTIwc3RyZWFtaW5nJTNEVHJ1ZSUyQyUwQSklMEElMEF2YWxpZF9kYXRhJTIwJTNEJTIwZGF0YXNldC50YWtlKDQwMDApJTBBdHJhaW5fZGF0YSUyMCUzRCUyMGRhdGFzZXQuc2tpcCg0MDAwKSUwQXRyYWluX2RhdGElMjAlM0QlMjB0cmFpbl9kYXRhLnNodWZmbGUoYnVmZmVyX3NpemUlM0Q1MDAwJTJDJTIwc2VlZCUzRFNFRUQp",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> tqdm <span class="hljs-keyword">import</span> tqdm


dataset = load_dataset(
    DATASET,
    data_dir=<span class="hljs-string">&quot;data&quot;</span>,
    split=<span class="hljs-string">&quot;train&quot;</span>,
    streaming=<span class="hljs-literal">True</span>,
)

valid_data = dataset.take(<span class="hljs-number">4000</span>)
train_data = dataset.skip(<span class="hljs-number">4000</span>)
train_data = train_data.shuffle(buffer_size=<span class="hljs-number">5000</span>, seed=SEED)`,wrap:!1}}),S=new C({props:{code:"dG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoTU9ERUwlMkMlMjB0cnVzdF9yZW1vdGVfY29kZSUzRFRydWUpJTBBJTBBJTBBZGVmJTIwY2hhcnNfdG9rZW5fcmF0aW8oZGF0YXNldCUyQyUyMHRva2VuaXplciUyQyUyMGRhdGFfY29sdW1uJTJDJTIwbmJfZXhhbXBsZXMlM0Q0MDApJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwRXN0aW1hdGUlMjB0aGUlMjBhdmVyYWdlJTIwbnVtYmVyJTIwb2YlMjBjaGFyYWN0ZXJzJTIwcGVyJTIwdG9rZW4lMjBpbiUyMHRoZSUyMGRhdGFzZXQuJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTBBJTIwJTIwJTIwJTIwdG90YWxfY2hhcmFjdGVycyUyQyUyMHRvdGFsX3Rva2VucyUyMCUzRCUyMDAlMkMlMjAwJTBBJTIwJTIwJTIwJTIwZm9yJTIwXyUyQyUyMGV4YW1wbGUlMjBpbiUyMHRxZG0oemlwKHJhbmdlKG5iX2V4YW1wbGVzKSUyQyUyMGl0ZXIoZGF0YXNldCkpJTJDJTIwdG90YWwlM0RuYl9leGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b3RhbF9jaGFyYWN0ZXJzJTIwJTJCJTNEJTIwbGVuKGV4YW1wbGUlNUJkYXRhX2NvbHVtbiU1RCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b3RhbF90b2tlbnMlMjAlMkIlM0QlMjBsZW4odG9rZW5pemVyKGV4YW1wbGUlNUJkYXRhX2NvbHVtbiU1RCkudG9rZW5zKCkpJTBBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwdG90YWxfY2hhcmFjdGVycyUyMCUyRiUyMHRvdGFsX3Rva2VucyUwQSUwQSUwQWNoYXJzX3Blcl90b2tlbiUyMCUzRCUyMGNoYXJzX3Rva2VuX3JhdGlvKHRyYWluX2RhdGElMkMlMjB0b2tlbml6ZXIlMkMlMjBEQVRBX0NPTFVNTiklMEFwcmludChmJTIyVGhlJTIwY2hhcmFjdGVyJTIwdG8lMjB0b2tlbiUyMHJhdGlvJTIwb2YlMjB0aGUlMjBkYXRhc2V0JTIwaXMlM0ElMjAlN0JjaGFyc19wZXJfdG9rZW4lM0EuMmYlN0QlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=<span class="hljs-literal">True</span>)


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">chars_token_ratio</span>(<span class="hljs-params">dataset, tokenizer, data_column, nb_examples=<span class="hljs-number">400</span></span>):
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;&quot;&quot;
<span class="hljs-meta">... </span>    Estimate the average number of characters per token in the dataset.
<span class="hljs-meta">... </span>    &quot;&quot;&quot;</span>

<span class="hljs-meta">... </span>    total_characters, total_tokens = <span class="hljs-number">0</span>, <span class="hljs-number">0</span>
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> _, example <span class="hljs-keyword">in</span> tqdm(<span class="hljs-built_in">zip</span>(<span class="hljs-built_in">range</span>(nb_examples), <span class="hljs-built_in">iter</span>(dataset)), total=nb_examples):
<span class="hljs-meta">... </span>        total_characters += <span class="hljs-built_in">len</span>(example[data_column])
<span class="hljs-meta">... </span>        total_tokens += <span class="hljs-built_in">len</span>(tokenizer(example[data_column]).tokens())

<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> total_characters / total_tokens


<span class="hljs-meta">&gt;&gt;&gt; </span>chars_per_token = chars_token_ratio(train_data, tokenizer, DATA_COLUMN)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;The character to token ratio of the dataset is: <span class="hljs-subst">{chars_per_token:<span class="hljs-number">.2</span>f}</span>&quot;</span>)`,wrap:!1}}),v=new C({props:{code:" ",highlighted:`<span class="hljs-keyword">import</span> functools
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np


<span class="hljs-comment"># Helper function to get token ids of the special tokens for prefix, suffix and middle for FIM transformations.</span>
<span class="hljs-meta">@functools.lru_cache(<span class="hljs-params">maxsize=<span class="hljs-literal">None</span></span>)</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_fim_token_ids</span>(<span class="hljs-params">tokenizer</span>):
    <span class="hljs-keyword">try</span>:
        FIM_PREFIX, FIM_MIDDLE, FIM_SUFFIX, FIM_PAD = tokenizer.special_tokens_map[<span class="hljs-string">&quot;additional_special_tokens&quot;</span>][<span class="hljs-number">1</span>:<span class="hljs-number">5</span>]
        suffix_tok_id, prefix_tok_id, middle_tok_id, pad_tok_id = (
            tokenizer.vocab[tok] <span class="hljs-keyword">for</span> tok <span class="hljs-keyword">in</span> [FIM_SUFFIX, FIM_PREFIX, FIM_MIDDLE, FIM_PAD]
        )
    <span class="hljs-keyword">except</span> KeyError:
        suffix_tok_id, prefix_tok_id, middle_tok_id, pad_tok_id = <span class="hljs-literal">None</span>, <span class="hljs-literal">None</span>, <span class="hljs-literal">None</span>, <span class="hljs-literal">None</span>
    <span class="hljs-keyword">return</span> suffix_tok_id, prefix_tok_id, middle_tok_id, pad_tok_id


<span class="hljs-comment">## Adapted from https://github.com/bigcode-project/Megatron-LM/blob/6c4bf908df8fd86b4977f54bf5b8bd4b521003d1/megatron/data/gpt_dataset.py</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">permute</span>(<span class="hljs-params">
    sample,
    np_rng,
    suffix_tok_id,
    prefix_tok_id,
    middle_tok_id,
    pad_tok_id,
    fim_rate=<span class="hljs-number">0.5</span>,
    fim_spm_rate=<span class="hljs-number">0.5</span>,
    truncate_or_pad=<span class="hljs-literal">False</span>,
</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Take in a sample (list of tokens) and perform a FIM transformation on it with a probability of fim_rate, using two FIM modes:
    PSM and SPM (with a probability of fim_spm_rate).
    &quot;&quot;&quot;</span>

    <span class="hljs-comment"># The if condition will trigger with the probability of fim_rate</span>
    <span class="hljs-comment"># This means FIM transformations will apply to samples with a probability of fim_rate</span>
    <span class="hljs-keyword">if</span> np_rng.binomial(<span class="hljs-number">1</span>, fim_rate):

        <span class="hljs-comment"># Split the sample into prefix, middle, and suffix, based on randomly generated indices stored in the boundaries list.</span>
        boundaries = <span class="hljs-built_in">list</span>(np_rng.randint(low=<span class="hljs-number">0</span>, high=<span class="hljs-built_in">len</span>(sample) + <span class="hljs-number">1</span>, size=<span class="hljs-number">2</span>))
        boundaries.sort()

        prefix = np.array(sample[: boundaries[<span class="hljs-number">0</span>]], dtype=np.int64)
        middle = np.array(sample[boundaries[<span class="hljs-number">0</span>] : boundaries[<span class="hljs-number">1</span>]], dtype=np.int64)
        suffix = np.array(sample[boundaries[<span class="hljs-number">1</span>] :], dtype=np.int64)

        <span class="hljs-keyword">if</span> truncate_or_pad:
            <span class="hljs-comment"># calculate the new total length of the sample, taking into account tokens indicating prefix, middle, and suffix</span>
            new_length = suffix.shape[<span class="hljs-number">0</span>] + prefix.shape[<span class="hljs-number">0</span>] + middle.shape[<span class="hljs-number">0</span>] + <span class="hljs-number">3</span>
            diff = new_length - <span class="hljs-built_in">len</span>(sample)

            <span class="hljs-comment"># trancate or pad if there&#x27;s a difference in length between the new length and the original</span>
            <span class="hljs-keyword">if</span> diff &gt; <span class="hljs-number">0</span>:
                <span class="hljs-keyword">if</span> suffix.shape[<span class="hljs-number">0</span>] &lt;= diff:
                    <span class="hljs-keyword">return</span> sample, np_rng
                suffix = suffix[: suffix.shape[<span class="hljs-number">0</span>] - diff]
            <span class="hljs-keyword">elif</span> diff &lt; <span class="hljs-number">0</span>:
                suffix = np.concatenate([suffix, np.full((-<span class="hljs-number">1</span> * diff), pad_tok_id)])

        <span class="hljs-comment"># With the probability of fim_spm_rateapply SPM variant of FIM transformations</span>
        <span class="hljs-comment"># SPM: suffix, prefix, middle</span>
        <span class="hljs-keyword">if</span> np_rng.binomial(<span class="hljs-number">1</span>, fim_spm_rate):
            new_sample = np.concatenate(
                [
                    [prefix_tok_id, suffix_tok_id],
                    suffix,
                    [middle_tok_id],
                    prefix,
                    middle,
                ]
            )
        <span class="hljs-comment"># Otherwise, apply the PSM variant of FIM transformations</span>
        <span class="hljs-comment"># PSM: prefix, suffix, middle</span>
        <span class="hljs-keyword">else</span>:

            new_sample = np.concatenate(
                [
                    [prefix_tok_id],
                    prefix,
                    [suffix_tok_id],
                    suffix,
                    [middle_tok_id],
                    middle,
                ]
            )
    <span class="hljs-keyword">else</span>:
        <span class="hljs-comment"># don&#x27;t apply FIM transformations</span>
        new_sample = sample

    <span class="hljs-keyword">return</span> <span class="hljs-built_in">list</span>(new_sample), np_rng`,wrap:!1}}),$=new C({props:{code:" ",highlighted:`<span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> IterableDataset
<span class="hljs-keyword">from</span> torch.utils.data.dataloader <span class="hljs-keyword">import</span> DataLoader
<span class="hljs-keyword">import</span> random

<span class="hljs-comment"># Create an Iterable dataset that returns constant-length chunks of tokens from a stream of text files.</span>


<span class="hljs-keyword">class</span> <span class="hljs-title class_">ConstantLengthDataset</span>(<span class="hljs-title class_ inherited__">IterableDataset</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Iterable dataset that returns constant length chunks of tokens from stream of text files.
        Args:
            tokenizer (Tokenizer): The processor used for proccessing the data.
            dataset (dataset.Dataset): Dataset with text files.
            infinite (bool): If True the iterator is reset after dataset reaches end else stops.
            seq_length (int): Length of token sequences to return.
            num_of_sequences (int): Number of token sequences to keep in buffer.
            chars_per_token (int): Number of characters per token used to estimate number of tokens in text buffer.
            fim_rate (float): Rate (0.0 to 1.0) that sample will be permuted with FIM.
            fim_spm_rate (float): Rate (0.0 to 1.0) of FIM permuations that will use SPM.
            seed (int): Seed for random number generator.
    &quot;&quot;&quot;</span>

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">
        self,
        tokenizer,
        dataset,
        infinite=<span class="hljs-literal">False</span>,
        seq_length=<span class="hljs-number">1024</span>,
        num_of_sequences=<span class="hljs-number">1024</span>,
        chars_per_token=<span class="hljs-number">3.6</span>,
        content_field=<span class="hljs-string">&quot;content&quot;</span>,
        fim_rate=<span class="hljs-number">0.5</span>,
        fim_spm_rate=<span class="hljs-number">0.5</span>,
        seed=<span class="hljs-number">0</span>,
    </span>):
        self.tokenizer = tokenizer
        self.concat_token_id = tokenizer.eos_token_id
        self.dataset = dataset
        self.seq_length = seq_length
        self.infinite = infinite
        self.current_size = <span class="hljs-number">0</span>
        self.max_buffer_size = seq_length * chars_per_token * num_of_sequences
        self.content_field = content_field
        self.fim_rate = fim_rate
        self.fim_spm_rate = fim_spm_rate
        self.seed = seed

        (
            self.suffix_tok_id,
            self.prefix_tok_id,
            self.middle_tok_id,
            self.pad_tok_id,
        ) = get_fim_token_ids(self.tokenizer)
        <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> self.suffix_tok_id <span class="hljs-keyword">and</span> self.fim_rate &gt; <span class="hljs-number">0</span>:
            <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;FIM is not supported by tokenizer, disabling FIM&quot;</span>)
            self.fim_rate = <span class="hljs-number">0</span>

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__iter__</span>(<span class="hljs-params">self</span>):
        iterator = <span class="hljs-built_in">iter</span>(self.dataset)
        more_examples = <span class="hljs-literal">True</span>
        np_rng = np.random.RandomState(seed=self.seed)
        <span class="hljs-keyword">while</span> more_examples:
            buffer, buffer_len = [], <span class="hljs-number">0</span>
            <span class="hljs-keyword">while</span> <span class="hljs-literal">True</span>:
                <span class="hljs-keyword">if</span> buffer_len &gt;= self.max_buffer_size:
                    <span class="hljs-keyword">break</span>
                <span class="hljs-keyword">try</span>:
                    buffer.append(<span class="hljs-built_in">next</span>(iterator)[self.content_field])
                    buffer_len += <span class="hljs-built_in">len</span>(buffer[-<span class="hljs-number">1</span>])
                <span class="hljs-keyword">except</span> StopIteration:
                    <span class="hljs-keyword">if</span> self.infinite:
                        iterator = <span class="hljs-built_in">iter</span>(self.dataset)
                    <span class="hljs-keyword">else</span>:
                        more_examples = <span class="hljs-literal">False</span>
                        <span class="hljs-keyword">break</span>
            tokenized_inputs = self.tokenizer(buffer, truncation=<span class="hljs-literal">False</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
            all_token_ids = []

            <span class="hljs-keyword">for</span> tokenized_input <span class="hljs-keyword">in</span> tokenized_inputs:
                <span class="hljs-comment"># optionally do FIM permutations</span>
                <span class="hljs-keyword">if</span> self.fim_rate &gt; <span class="hljs-number">0</span>:
                    tokenized_input, np_rng = permute(
                        tokenized_input,
                        np_rng,
                        self.suffix_tok_id,
                        self.prefix_tok_id,
                        self.middle_tok_id,
                        self.pad_tok_id,
                        fim_rate=self.fim_rate,
                        fim_spm_rate=self.fim_spm_rate,
                        truncate_or_pad=<span class="hljs-literal">False</span>,
                    )

                all_token_ids.extend(tokenized_input + [self.concat_token_id])
            examples = []
            <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">len</span>(all_token_ids), self.seq_length):
                input_ids = all_token_ids[i : i + self.seq_length]
                <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(input_ids) == self.seq_length:
                    examples.append(input_ids)
            random.shuffle(examples)
            <span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> examples:
                self.current_size += <span class="hljs-number">1</span>
                <span class="hljs-keyword">yield</span> {
                    <span class="hljs-string">&quot;input_ids&quot;</span>: torch.LongTensor(example),
                    <span class="hljs-string">&quot;labels&quot;</span>: torch.LongTensor(example),
                }


train_dataset = ConstantLengthDataset(
    tokenizer,
    train_data,
    infinite=<span class="hljs-literal">True</span>,
    seq_length=SEQ_LENGTH,
    chars_per_token=chars_per_token,
    content_field=DATA_COLUMN,
    fim_rate=FIM_RATE,
    fim_spm_rate=FIM_SPM_RATE,
    seed=SEED,
)
eval_dataset = ConstantLengthDataset(
    tokenizer,
    valid_data,
    infinite=<span class="hljs-literal">False</span>,
    seq_length=SEQ_LENGTH,
    chars_per_token=chars_per_token,
    content_field=DATA_COLUMN,
    fim_rate=FIM_RATE,
    fim_spm_rate=FIM_SPM_RATE,
    seed=SEED,
)`,wrap:!1}}),L=new Vl({props:{title:"准备模型",local:"准备模型",headingTag:"h2"}}),Ml=new C({props:{code:"ZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb3JhQ29uZmlnJTJDJTIwZ2V0X3BlZnRfbW9kZWwlMkMlMjBwcmVwYXJlX21vZGVsX2Zvcl9rYml0X3RyYWluaW5nJTBBZnJvbSUyMHBlZnQudHVuZXJzLmxvcmElMjBpbXBvcnQlMjBMb3JhTGF5ZXIlMEElMEFsb2FkX2luXzhiaXQlMjAlM0QlMjBGYWxzZSUwQSUwQSUyMyUyMDQtYml0JTIwcXVhbnRpemF0aW9uJTBBY29tcHV0ZV9kdHlwZSUyMCUzRCUyMGdldGF0dHIodG9yY2glMkMlMjBCTkJfNEJJVF9DT01QVVRFX0RUWVBFKSUwQSUwQWJuYl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEElMjAlMjAlMjAlMjBibmJfNGJpdF9jb21wdXRlX2R0eXBlJTNEY29tcHV0ZV9kdHlwZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3VzZV9kb3VibGVfcXVhbnQlM0RVU0VfTkVTVEVEX1FVQU5UJTJDJTBBKSUwQSUwQWRldmljZV9tYXAlMjAlM0QlMjAlN0IlMjIlMjIlM0ElMjAwJTdEJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwTU9ERUwlMkMlMEElMjAlMjAlMjAlMjBsb2FkX2luXzhiaXQlM0Rsb2FkX2luXzhiaXQlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEYm5iX2NvbmZpZyUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0RkZXZpY2VfbWFwJTJDJTBBJTIwJTIwJTIwJTIwdXNlX2NhY2hlJTNERmFsc2UlMkMlMjAlMjAlMjMlMjBXZSUyMHdpbGwlMjBiZSUyMHVzaW5nJTIwZ3JhZGllbnQlMjBjaGVja3BvaW50aW5nJTBBJTIwJTIwJTIwJTIwdHJ1c3RfcmVtb3RlX2NvZGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwdXNlX2ZsYXNoX2F0dGVudGlvbl8yJTNEVHJ1ZSUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> LoraConfig, get_peft_model, prepare_model_for_kbit_training
<span class="hljs-keyword">from</span> peft.tuners.lora <span class="hljs-keyword">import</span> LoraLayer

load_in_8bit = <span class="hljs-literal">False</span>

<span class="hljs-comment"># 4-bit quantization</span>
compute_dtype = <span class="hljs-built_in">getattr</span>(torch, BNB_4BIT_COMPUTE_DTYPE)

bnb_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
    bnb_4bit_compute_dtype=compute_dtype,
    bnb_4bit_use_double_quant=USE_NESTED_QUANT,
)

device_map = {<span class="hljs-string">&quot;&quot;</span>: <span class="hljs-number">0</span>}

model = AutoModelForCausalLM.from_pretrained(
    MODEL,
    load_in_8bit=load_in_8bit,
    quantization_config=bnb_config,
    device_map=device_map,
    use_cache=<span class="hljs-literal">False</span>,  <span class="hljs-comment"># We will be using gradient checkpointing</span>
    trust_remote_code=<span class="hljs-literal">True</span>,
    use_flash_attention_2=<span class="hljs-literal">True</span>,
)`,wrap:!1}}),el=new C({props:{code:"bW9kZWwlMjAlM0QlMjBwcmVwYXJlX21vZGVsX2Zvcl9rYml0X3RyYWluaW5nKG1vZGVsKQ==",highlighted:"model = prepare_model_for_kbit_training(model)",wrap:!1}}),yl=new C({props:{code:"JTIzJTIwU2V0JTIwdXAlMjBsb3JhJTBBcGVmdF9jb25maWclMjAlM0QlMjBMb3JhQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvcmFfYWxwaGElM0RMT1JBX0FMUEhBJTJDJTBBJTIwJTIwJTIwJTIwbG9yYV9kcm9wb3V0JTNETE9SQV9EUk9QT1VUJTJDJTBBJTIwJTIwJTIwJTIwciUzRExPUkFfUiUyQyUwQSUyMCUyMCUyMCUyMGJpYXMlM0QlMjJub25lJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFza190eXBlJTNEJTIyQ0FVU0FMX0xNJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFyZ2V0X21vZHVsZXMlM0RMT1JBX1RBUkdFVF9NT0RVTEVTLnNwbGl0KCUyMiUyQyUyMiklMkMlMEEpJTBBJTBBbW9kZWwlMjAlM0QlMjBnZXRfcGVmdF9tb2RlbChtb2RlbCUyQyUyMHBlZnRfY29uZmlnKSUwQW1vZGVsLnByaW50X3RyYWluYWJsZV9wYXJhbWV0ZXJzKCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Set up lora</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>peft_config = LoraConfig(
<span class="hljs-meta">... </span>    lora_alpha=LORA_ALPHA,
<span class="hljs-meta">... </span>    lora_dropout=LORA_DROPOUT,
<span class="hljs-meta">... </span>    r=LORA_R,
<span class="hljs-meta">... </span>    bias=<span class="hljs-string">&quot;none&quot;</span>,
<span class="hljs-meta">... </span>    task_type=<span class="hljs-string">&quot;CAUSAL_LM&quot;</span>,
<span class="hljs-meta">... </span>    target_modules=LORA_TARGET_MODULES.split(<span class="hljs-string">&quot;,&quot;</span>),
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model = get_peft_model(model, peft_config)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.print_trainable_parameters()`,wrap:!1}}),Tl=new Vl({props:{title:"训练模型",local:"训练模型",headingTag:"h2"}}),Il=new C({props:{code:"dHJhaW5fZGF0YS5zdGFydF9pdGVyYXRpb24lMjAlM0QlMjAwJTBBJTBBJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0RmJTIyWW91cl9IRl91c2VybmFtZSUyRiU3Qk9VVFBVVF9ESVIlN0QlMjIlMkMlMEElMjAlMjAlMjAlMjBkYXRhbG9hZGVyX2Ryb3BfbGFzdCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBldmFsdWF0aW9uX3N0cmF0ZWd5JTNEJTIyc3RlcHMlMjIlMkMlMEElMjAlMjAlMjAlMjBzYXZlX3N0cmF0ZWd5JTNEJTIyc3RlcHMlMjIlMkMlMEElMjAlMjAlMjAlMjBtYXhfc3RlcHMlM0RNQVhfU1RFUFMlMkMlMEElMjAlMjAlMjAlMjBldmFsX3N0ZXBzJTNERVZBTF9GUkVRJTJDJTBBJTIwJTIwJTIwJTIwc2F2ZV9zdGVwcyUzRFNBVkVfRlJFUSUyQyUwQSUyMCUyMCUyMCUyMGxvZ2dpbmdfc3RlcHMlM0RMT0dfRlJFUSUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzREJBVENIX1NJWkUlMkMlMEElMjAlMjAlMjAlMjBwZXJfZGV2aWNlX2V2YWxfYmF0Y2hfc2l6ZSUzREJBVENIX1NJWkUlMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNETFIlMkMlMEElMjAlMjAlMjAlMjBscl9zY2hlZHVsZXJfdHlwZSUzRExSX1NDSEVEVUxFUl9UWVBFJTJDJTBBJTIwJTIwJTIwJTIwd2FybXVwX3N0ZXBzJTNETlVNX1dBUk1VUF9TVEVQUyUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzREdSX0FDQ19TVEVQUyUyQyUwQSUyMCUyMCUyMCUyMGdyYWRpZW50X2NoZWNrcG9pbnRpbmclM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwZnAxNiUzREZQMTYlMkMlMEElMjAlMjAlMjAlMjBiZjE2JTNEQkYxNiUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodF9kZWNheSUzRFdFSUdIVF9ERUNBWSUyQyUwQSUyMCUyMCUyMCUyMHB1c2hfdG9faHViJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGluY2x1ZGVfdG9rZW5zX3Blcl9zZWNvbmQlM0RUcnVlJTJDJTBBKQ==",highlighted:`train_data.start_iteration = <span class="hljs-number">0</span>


training_args = TrainingArguments(
    output_dir=<span class="hljs-string">f&quot;Your_HF_username/<span class="hljs-subst">{OUTPUT_DIR}</span>&quot;</span>,
    dataloader_drop_last=<span class="hljs-literal">True</span>,
    evaluation_strategy=<span class="hljs-string">&quot;steps&quot;</span>,
    save_strategy=<span class="hljs-string">&quot;steps&quot;</span>,
    max_steps=MAX_STEPS,
    eval_steps=EVAL_FREQ,
    save_steps=SAVE_FREQ,
    logging_steps=LOG_FREQ,
    per_device_train_batch_size=BATCH_SIZE,
    per_device_eval_batch_size=BATCH_SIZE,
    learning_rate=LR,
    lr_scheduler_type=LR_SCHEDULER_TYPE,
    warmup_steps=NUM_WARMUP_STEPS,
    gradient_accumulation_steps=GR_ACC_STEPS,
    gradient_checkpointing=<span class="hljs-literal">True</span>,
    fp16=FP16,
    bf16=BF16,
    weight_decay=WEIGHT_DECAY,
    push_to_hub=<span class="hljs-literal">True</span>,
    include_tokens_per_second=<span class="hljs-literal">True</span>,
)`,wrap:!1}}),il=new C({props:{code:"dHJhaW5lciUyMCUzRCUyMFRyYWluZXIobW9kZWwlM0Rtb2RlbCUyQyUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzJTJDJTIwdHJhaW5fZGF0YXNldCUzRHRyYWluX2RhdGFzZXQlMkMlMjBldmFsX2RhdGFzZXQlM0RldmFsX2RhdGFzZXQpJTBBJTBBcHJpbnQoJTIyVHJhaW5pbmcuLi4lMjIpJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(model=model, args=training_args, train_dataset=train_dataset, eval_dataset=eval_dataset)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Training...&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`,wrap:!1}}),cl=new C({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:"trainer.push_to_hub()",wrap:!1}}),dl=new Vl({props:{title:"推理",local:"推理",headingTag:"h2"}}),rl=new C({props:{code:"ZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBQZWZ0TW9kZWwlMEFpbXBvcnQlMjB0b3JjaCUwQSUwQSUyMyUyMGxvYWQlMjB0aGUlMjBvcmlnaW5hbCUyMG1vZGVsJTIwZmlyc3QlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChNT0RFTCUyQyUyMHRydXN0X3JlbW90ZV9jb2RlJTNEVHJ1ZSklMEFiYXNlX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyME1PREVMJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRE5vbmUlMkMlMEElMjAlMjAlMjAlMjBkZXZpY2VfbWFwJTNETm9uZSUyQyUwQSUyMCUyMCUyMCUyMHRydXN0X3JlbW90ZV9jb2RlJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMEEpLmN1ZGEoKSUwQSUwQSUyMyUyMG1lcmdlJTIwZmluZS10dW5lZCUyMHdlaWdodHMlMjB3aXRoJTIwdGhlJTIwYmFzZSUyMG1vZGVsJTBBcGVmdF9tb2RlbF9pZCUyMCUzRCUyMGYlMjJZb3VyX0hGX3VzZXJuYW1lJTJGJTdCT1VUUFVUX0RJUiU3RCUyMiUwQW1vZGVsJTIwJTNEJTIwUGVmdE1vZGVsLmZyb21fcHJldHJhaW5lZChiYXNlX21vZGVsJTJDJTIwcGVmdF9tb2RlbF9pZCklMEFtb2RlbC5tZXJnZV9hbmRfdW5sb2FkKCk=",highlighted:`<span class="hljs-keyword">from</span> peft <span class="hljs-keyword">import</span> PeftModel
<span class="hljs-keyword">import</span> torch

<span class="hljs-comment"># load the original model first</span>
tokenizer = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=<span class="hljs-literal">True</span>)
base_model = AutoModelForCausalLM.from_pretrained(
    MODEL,
    quantization_config=<span class="hljs-literal">None</span>,
    device_map=<span class="hljs-literal">None</span>,
    trust_remote_code=<span class="hljs-literal">True</span>,
    torch_dtype=torch.bfloat16,
).cuda()

<span class="hljs-comment"># merge fine-tuned weights with the base model</span>
peft_model_id = <span class="hljs-string">f&quot;Your_HF_username/<span class="hljs-subst">{OUTPUT_DIR}</span>&quot;</span>
model = PeftModel.from_pretrained(base_model, peft_model_id)
model.merge_and_unload()`,wrap:!1}}),fl=new C({props:{code:"ZGVmJTIwZ2V0X2NvZGVfY29tcGxldGlvbihwcmVmaXglMkMlMjBzdWZmaXgpJTNBJTBBJTIwJTIwJTIwJTIwdGV4dCUyMCUzRCUyMHByb21wdCUyMCUzRCUyMGYlMjIlMjIlMjIlM0NmaW1fcHJlZml4JTNFJTdCcHJlZml4JTdEJTNDZmltX3N1ZmZpeCUzRSU3QnN1ZmZpeCU3RCUzQ2ZpbV9taWRkbGUlM0UlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjBtb2RlbC5ldmFsKCklMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW5wdXRfaWRzJTNEdG9rZW5pemVyKHRleHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS5pbnB1dF9pZHMuY3VkYSgpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbWF4X25ld190b2tlbnMlM0QxMjglMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0ZW1wZXJhdHVyZSUzRDAuMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRvcF9rJTNENTAlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b3BfcCUzRDAuOTUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBkb19zYW1wbGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVwZXRpdGlvbl9wZW5hbHR5JTNEMS4wJTJDJTBBJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHRva2VuaXplci5iYXRjaF9kZWNvZGUob3V0cHV0cyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQ=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_code_completion</span>(<span class="hljs-params">prefix, suffix</span>):
    text = prompt = <span class="hljs-string">f&quot;&quot;&quot;&lt;fim_prefix&gt;<span class="hljs-subst">{prefix}</span>&lt;fim_suffix&gt;<span class="hljs-subst">{suffix}</span>&lt;fim_middle&gt;&quot;&quot;&quot;</span>
    model.<span class="hljs-built_in">eval</span>()
    outputs = model.generate(
        input_ids=tokenizer(text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_ids.cuda(),
        max_new_tokens=<span class="hljs-number">128</span>,
        temperature=<span class="hljs-number">0.2</span>,
        top_k=<span class="hljs-number">50</span>,
        top_p=<span class="hljs-number">0.95</span>,
        do_sample=<span class="hljs-literal">True</span>,
        repetition_penalty=<span class="hljs-number">1.0</span>,
    )
    <span class="hljs-keyword">return</span> tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]`,wrap:!1}}),hl=new C({props:{code:"cHJlZml4JTIwJTNEJTIwJTIyJTIyJTIyZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb3JhQ29uZmlnJTJDJTIwVGFza1R5cGUlMkMlMjBnZXRfcGVmdF9tb2RlbCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQXBlZnRfY29uZmlnJTIwJTNEJTIwTG9yYUNvbmZpZyglMEElMjIlMjIlMjIlMEFzdWZmaXglMjAlM0QlMjAlMjIlMjIlMjIlMjIlMjIlMjIlMEElMEFwcmludChnZXRfY29kZV9jb21wbGV0aW9uKHByZWZpeCUyQyUyMHN1ZmZpeCkp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>prefix = <span class="hljs-string">&quot;&quot;&quot;from peft import LoraConfig, TaskType, get_peft_model
<span class="hljs-meta">... </span>from transformers import AutoModelForCausalLM
<span class="hljs-meta">... </span>peft_config = LoraConfig(
<span class="hljs-meta">... </span>&quot;&quot;&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>suffix = <span class="hljs-string">&quot;&quot;&quot;&quot;&quot;&quot;</span>

<span class="hljs-meta">... </span><span class="hljs-built_in">print</span>(get_code_completion(prefix, suffix))`,wrap:!1}}),Gl=new C({props:{code:"cHJlZml4JTIwJTNEJTIwJTIyJTIyJTIyZnJvbSUyMHBlZnQlMjBpbXBvcnQlMjBMb3JhQ29uZmlnJTJDJTIwVGFza1R5cGUlMkMlMjBnZXRfcGVmdF9tb2RlbCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQXBlZnRfY29uZmlnJTIwJTNEJTIwTG9yYUNvbmZpZyglMEElMjIlMjIlMjIlMEFzdWZmaXglMjAlM0QlMjAlMjIlMjIlMjIlMjIlMjIlMjIlMEElMEFwcmludChnZXRfY29kZV9jb21wbGV0aW9uKHByZWZpeCUyQyUyMHN1ZmZpeCkp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>prefix = <span class="hljs-string">&quot;&quot;&quot;from peft import LoraConfig, TaskType, get_peft_model
<span class="hljs-meta">... </span>from transformers import AutoModelForCausalLM
<span class="hljs-meta">... </span>peft_config = LoraConfig(
<span class="hljs-meta">... </span>&quot;&quot;&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>suffix = <span class="hljs-string">&quot;&quot;&quot;&quot;&quot;&quot;</span>

<span class="hljs-meta">... </span><span class="hljs-built_in">print</span>(get_code_completion(prefix, suffix))`,wrap:!1}}),Xl=new ye({props:{source:"https://github.com/huggingface/cookbook/blob/main/notebooks/zh-CN/fine_tuning_code_llm_on_single_gpu.md"}}),{c(){i=n("meta"),Wl=t(),Ql=n("p"),Sl=t(),J(m.$$.fragment),Nl=t(),J(o.$$.fragment),gl=t(),c=n("p"),c.innerHTML=ns,Yl=t(),d=n("p"),d.textContent=ys,Hl=t(),A=n("p"),A.textContent=Us,vl=t(),J(r.$$.fragment),xl=t(),b=n("p"),b.innerHTML=Js,zl=t(),J(f.$$.fragment),$l=t(),u=n("p"),u.innerHTML=Ts,Ll=t(),J(h.$$.fragment),Dl=t(),Z=n("p"),Z.innerHTML=js,ql=t(),B=n("p"),B.textContent=ps,Pl=t(),J(_.$$.fragment),Kl=t(),G=n("p"),G.textContent=ws,Ol=t(),J(R.$$.fragment),lM=t(),J(E.$$.fragment),MM=t(),J(k.$$.fragment),sM=t(),X=n("p"),X.textContent=Is,eM=t(),V=n("p"),V.textContent=Cs,tM=t(),J(Q.$$.fragment),aM=t(),F=n("p"),F.textContent=is,nM=t(),W=n("p"),W.innerHTML=ms,yM=t(),J(S.$$.fragment),UM=t(),N=n("pre"),N.textContent=os,JM=t(),g=n("p"),g.textContent=cs,TM=t(),Y=n("p"),Y.innerHTML=ds,jM=t(),H=n("p"),H.innerHTML=As,pM=t(),J(v.$$.fragment),wM=t(),x=n("p"),x.innerHTML=rs,IM=t(),z=n("p"),z.innerHTML=bs,CM=t(),J($.$$.fragment),iM=t(),J(L.$$.fragment),mM=t(),D=n("p"),D.textContent=fs,oM=t(),q=n("p"),q.innerHTML=us,cM=t(),P=n("p"),P.innerHTML=hs,dM=t(),K=n("p"),K.innerHTML=Zs,AM=t(),O=n("p"),O.innerHTML=Bs,rM=t(),ll=n("p"),ll.innerHTML=_s,bM=t(),J(Ml.$$.fragment),fM=t(),sl=n("p"),sl.innerHTML=Gs,uM=t(),J(el.$$.fragment),hM=t(),tl=n("p"),tl.textContent=Rs,ZM=t(),al=n("p"),al.innerHTML=Es,BM=t(),nl=n("p"),nl.innerHTML=ks,_M=t(),J(yl.$$.fragment),GM=t(),Ul=n("pre"),Ul.textContent=Xs,RM=t(),Jl=n("p"),Jl.textContent=Vs,EM=t(),J(Tl.$$.fragment),kM=t(),jl=n("p"),jl.textContent=Qs,XM=t(),pl=n("p"),pl.innerHTML=Fs,VM=t(),wl=n("p"),wl.textContent=Ws,QM=t(),J(Il.$$.fragment),FM=t(),Cl=n("p"),Cl.innerHTML=Ss,WM=t(),J(il.$$.fragment),SM=t(),ml=n("pre"),ml.textContent=Ns,NM=t(),ol=n("p"),ol.textContent=gs,gM=t(),J(cl.$$.fragment),YM=t(),J(dl.$$.fragment),HM=t(),Al=n("p"),Al.textContent=Ys,vM=t(),J(rl.$$.fragment),xM=t(),bl=n("p"),bl.innerHTML=Hs,zM=t(),J(fl.$$.fragment),$M=t(),ul=n("p"),ul.innerHTML=vs,LM=t(),J(hl.$$.fragment),DM=t(),Zl=n("pre"),Zl.textContent=xs,qM=t(),Bl=n("p"),Bl.innerHTML=zs,PM=t(),_l=n("p"),_l.textContent=$s,KM=t(),J(Gl.$$.fragment),OM=t(),Rl=n("pre"),Rl.textContent=Ls,ls=t(),El=n("p"),El.innerHTML=Ds,Ms=t(),kl=n("p"),kl.innerHTML=qs,ss=t(),J(Xl.$$.fragment),es=t(),Fl=n("p"),this.h()},l(l){const M=te("svelte-u9bgzb",document.head);i=y(M,"META",{name:!0,content:!0}),M.forEach(s),Wl=a(l),Ql=y(l,"P",{}),Ps(Ql).forEach(s),Sl=a(l),T(m.$$.fragment,l),Nl=a(l),T(o.$$.fragment,l),gl=a(l),c=y(l,"P",{"data-svelte-h":!0}),U(c)!=="svelte-1dsumsc"&&(c.innerHTML=ns),Yl=a(l),d=y(l,"P",{"data-svelte-h":!0}),U(d)!=="svelte-q659zy"&&(d.textContent=ys),Hl=a(l),A=y(l,"P",{"data-svelte-h":!0}),U(A)!=="svelte-7d3r9t"&&(A.textContent=Us),vl=a(l),T(r.$$.fragment,l),xl=a(l),b=y(l,"P",{"data-svelte-h":!0}),U(b)!=="svelte-16d0rk8"&&(b.innerHTML=Js),zl=a(l),T(f.$$.fragment,l),$l=a(l),u=y(l,"P",{"data-svelte-h":!0}),U(u)!=="svelte-1lmoyae"&&(u.innerHTML=Ts),Ll=a(l),T(h.$$.fragment,l),Dl=a(l),Z=y(l,"P",{"data-svelte-h":!0}),U(Z)!=="svelte-pqckta"&&(Z.innerHTML=js),ql=a(l),B=y(l,"P",{"data-svelte-h":!0}),U(B)!=="svelte-1bfizik"&&(B.textContent=ps),Pl=a(l),T(_.$$.fragment,l),Kl=a(l),G=y(l,"P",{"data-svelte-h":!0}),U(G)!=="svelte-q7qesu"&&(G.textContent=ws),Ol=a(l),T(R.$$.fragment,l),lM=a(l),T(E.$$.fragment,l),MM=a(l),T(k.$$.fragment,l),sM=a(l),X=y(l,"P",{"data-svelte-h":!0}),U(X)!=="svelte-nbrbdm"&&(X.textContent=Is),eM=a(l),V=y(l,"P",{"data-svelte-h":!0}),U(V)!=="svelte-156cg1h"&&(V.textContent=Cs),tM=a(l),T(Q.$$.fragment,l),aM=a(l),F=y(l,"P",{"data-svelte-h":!0}),U(F)!=="svelte-w6f819"&&(F.textContent=is),nM=a(l),W=y(l,"P",{"data-svelte-h":!0}),U(W)!=="svelte-11jtxw5"&&(W.innerHTML=ms),yM=a(l),T(S.$$.fragment,l),UM=a(l),N=y(l,"PRE",{"data-svelte-h":!0}),U(N)!=="svelte-ko6dme"&&(N.textContent=os),JM=a(l),g=y(l,"P",{"data-svelte-h":!0}),U(g)!=="svelte-qkug4j"&&(g.textContent=cs),TM=a(l),Y=y(l,"P",{"data-svelte-h":!0}),U(Y)!=="svelte-10u5js8"&&(Y.innerHTML=ds),jM=a(l),H=y(l,"P",{"data-svelte-h":!0}),U(H)!=="svelte-a6uov0"&&(H.innerHTML=As),pM=a(l),T(v.$$.fragment,l),wM=a(l),x=y(l,"P",{"data-svelte-h":!0}),U(x)!=="svelte-19bf5mc"&&(x.innerHTML=rs),IM=a(l),z=y(l,"P",{"data-svelte-h":!0}),U(z)!=="svelte-1kw27m4"&&(z.innerHTML=bs),CM=a(l),T($.$$.fragment,l),iM=a(l),T(L.$$.fragment,l),mM=a(l),D=y(l,"P",{"data-svelte-h":!0}),U(D)!=="svelte-1r3du7w"&&(D.textContent=fs),oM=a(l),q=y(l,"P",{"data-svelte-h":!0}),U(q)!=="svelte-2im68f"&&(q.innerHTML=us),cM=a(l),P=y(l,"P",{"data-svelte-h":!0}),U(P)!=="svelte-12xf50n"&&(P.innerHTML=hs),dM=a(l),K=y(l,"P",{"data-svelte-h":!0}),U(K)!=="svelte-n8dwtj"&&(K.innerHTML=Zs),AM=a(l),O=y(l,"P",{"data-svelte-h":!0}),U(O)!=="svelte-1frw6v8"&&(O.innerHTML=Bs),rM=a(l),ll=y(l,"P",{"data-svelte-h":!0}),U(ll)!=="svelte-d91ns3"&&(ll.innerHTML=_s),bM=a(l),T(Ml.$$.fragment,l),fM=a(l),sl=y(l,"P",{"data-svelte-h":!0}),U(sl)!=="svelte-1xl6sjo"&&(sl.innerHTML=Gs),uM=a(l),T(el.$$.fragment,l),hM=a(l),tl=y(l,"P",{"data-svelte-h":!0}),U(tl)!=="svelte-1eyziea"&&(tl.textContent=Rs),ZM=a(l),al=y(l,"P",{"data-svelte-h":!0}),U(al)!=="svelte-m5kbu0"&&(al.innerHTML=Es),BM=a(l),nl=y(l,"P",{"data-svelte-h":!0}),U(nl)!=="svelte-ag7n3w"&&(nl.innerHTML=ks),_M=a(l),T(yl.$$.fragment,l),GM=a(l),Ul=y(l,"PRE",{"data-svelte-h":!0}),U(Ul)!=="svelte-hpf32x"&&(Ul.textContent=Xs),RM=a(l),Jl=y(l,"P",{"data-svelte-h":!0}),U(Jl)!=="svelte-23hvvu"&&(Jl.textContent=Vs),EM=a(l),T(Tl.$$.fragment,l),kM=a(l),jl=y(l,"P",{"data-svelte-h":!0}),U(jl)!=="svelte-tja9vl"&&(jl.textContent=Qs),XM=a(l),pl=y(l,"P",{"data-svelte-h":!0}),U(pl)!=="svelte-yjq99r"&&(pl.innerHTML=Fs),VM=a(l),wl=y(l,"P",{"data-svelte-h":!0}),U(wl)!=="svelte-12swt94"&&(wl.textContent=Ws),QM=a(l),T(Il.$$.fragment,l),FM=a(l),Cl=y(l,"P",{"data-svelte-h":!0}),U(Cl)!=="svelte-emgn9w"&&(Cl.innerHTML=Ss),WM=a(l),T(il.$$.fragment,l),SM=a(l),ml=y(l,"PRE",{"data-svelte-h":!0}),U(ml)!=="svelte-9xzew6"&&(ml.textContent=Ns),NM=a(l),ol=y(l,"P",{"data-svelte-h":!0}),U(ol)!=="svelte-e3wwxu"&&(ol.textContent=gs),gM=a(l),T(cl.$$.fragment,l),YM=a(l),T(dl.$$.fragment,l),HM=a(l),Al=y(l,"P",{"data-svelte-h":!0}),U(Al)!=="svelte-1rt49jd"&&(Al.textContent=Ys),vM=a(l),T(rl.$$.fragment,l),xM=a(l),bl=y(l,"P",{"data-svelte-h":!0}),U(bl)!=="svelte-1tevq0n"&&(bl.innerHTML=Hs),zM=a(l),T(fl.$$.fragment,l),$M=a(l),ul=y(l,"P",{"data-svelte-h":!0}),U(ul)!=="svelte-vmzo1"&&(ul.innerHTML=vs),LM=a(l),T(hl.$$.fragment,l),DM=a(l),Zl=y(l,"PRE",{"data-svelte-h":!0}),U(Zl)!=="svelte-11ixuz2"&&(Zl.textContent=xs),qM=a(l),Bl=y(l,"P",{"data-svelte-h":!0}),U(Bl)!=="svelte-1s5jsz7"&&(Bl.innerHTML=zs),PM=a(l),_l=y(l,"P",{"data-svelte-h":!0}),U(_l)!=="svelte-10itto7"&&(_l.textContent=$s),KM=a(l),T(Gl.$$.fragment,l),OM=a(l),Rl=y(l,"PRE",{"data-svelte-h":!0}),U(Rl)!=="svelte-1ud779o"&&(Rl.textContent=Ls),ls=a(l),El=y(l,"P",{"data-svelte-h":!0}),U(El)!=="svelte-3s10dd"&&(El.innerHTML=Ds),Ms=a(l),kl=y(l,"P",{"data-svelte-h":!0}),U(kl)!=="svelte-1x2bp5"&&(kl.innerHTML=qs),ss=a(l),T(Xl.$$.fragment,l),es=a(l),Fl=y(l,"P",{}),Ps(Fl).forEach(s),this.h()},h(){Ks(i,"name","hf:doc:metadata"),Ks(i,"content",Je)},m(l,M){ae(document.head,i),e(l,Wl,M),e(l,Ql,M),e(l,Sl,M),j(m,l,M),e(l,Nl,M),j(o,l,M),e(l,gl,M),e(l,c,M),e(l,Yl,M),e(l,d,M),e(l,Hl,M),e(l,A,M),e(l,vl,M),j(r,l,M),e(l,xl,M),e(l,b,M),e(l,zl,M),j(f,l,M),e(l,$l,M),e(l,u,M),e(l,Ll,M),j(h,l,M),e(l,Dl,M),e(l,Z,M),e(l,ql,M),e(l,B,M),e(l,Pl,M),j(_,l,M),e(l,Kl,M),e(l,G,M),e(l,Ol,M),j(R,l,M),e(l,lM,M),j(E,l,M),e(l,MM,M),j(k,l,M),e(l,sM,M),e(l,X,M),e(l,eM,M),e(l,V,M),e(l,tM,M),j(Q,l,M),e(l,aM,M),e(l,F,M),e(l,nM,M),e(l,W,M),e(l,yM,M),j(S,l,M),e(l,UM,M),e(l,N,M),e(l,JM,M),e(l,g,M),e(l,TM,M),e(l,Y,M),e(l,jM,M),e(l,H,M),e(l,pM,M),j(v,l,M),e(l,wM,M),e(l,x,M),e(l,IM,M),e(l,z,M),e(l,CM,M),j($,l,M),e(l,iM,M),j(L,l,M),e(l,mM,M),e(l,D,M),e(l,oM,M),e(l,q,M),e(l,cM,M),e(l,P,M),e(l,dM,M),e(l,K,M),e(l,AM,M),e(l,O,M),e(l,rM,M),e(l,ll,M),e(l,bM,M),j(Ml,l,M),e(l,fM,M),e(l,sl,M),e(l,uM,M),j(el,l,M),e(l,hM,M),e(l,tl,M),e(l,ZM,M),e(l,al,M),e(l,BM,M),e(l,nl,M),e(l,_M,M),j(yl,l,M),e(l,GM,M),e(l,Ul,M),e(l,RM,M),e(l,Jl,M),e(l,EM,M),j(Tl,l,M),e(l,kM,M),e(l,jl,M),e(l,XM,M),e(l,pl,M),e(l,VM,M),e(l,wl,M),e(l,QM,M),j(Il,l,M),e(l,FM,M),e(l,Cl,M),e(l,WM,M),j(il,l,M),e(l,SM,M),e(l,ml,M),e(l,NM,M),e(l,ol,M),e(l,gM,M),j(cl,l,M),e(l,YM,M),j(dl,l,M),e(l,HM,M),e(l,Al,M),e(l,vM,M),j(rl,l,M),e(l,xM,M),e(l,bl,M),e(l,zM,M),j(fl,l,M),e(l,$M,M),e(l,ul,M),e(l,LM,M),j(hl,l,M),e(l,DM,M),e(l,Zl,M),e(l,qM,M),e(l,Bl,M),e(l,PM,M),e(l,_l,M),e(l,KM,M),j(Gl,l,M),e(l,OM,M),e(l,Rl,M),e(l,ls,M),e(l,El,M),e(l,Ms,M),e(l,kl,M),e(l,ss,M),j(Xl,l,M),e(l,es,M),e(l,Fl,M),ts=!0},p:le,i(l){ts||(p(m.$$.fragment,l),p(o.$$.fragment,l),p(r.$$.fragment,l),p(f.$$.fragment,l),p(h.$$.fragment,l),p(_.$$.fragment,l),p(R.$$.fragment,l),p(E.$$.fragment,l),p(k.$$.fragment,l),p(Q.$$.fragment,l),p(S.$$.fragment,l),p(v.$$.fragment,l),p($.$$.fragment,l),p(L.$$.fragment,l),p(Ml.$$.fragment,l),p(el.$$.fragment,l),p(yl.$$.fragment,l),p(Tl.$$.fragment,l),p(Il.$$.fragment,l),p(il.$$.fragment,l),p(cl.$$.fragment,l),p(dl.$$.fragment,l),p(rl.$$.fragment,l),p(fl.$$.fragment,l),p(hl.$$.fragment,l),p(Gl.$$.fragment,l),p(Xl.$$.fragment,l),ts=!0)},o(l){w(m.$$.fragment,l),w(o.$$.fragment,l),w(r.$$.fragment,l),w(f.$$.fragment,l),w(h.$$.fragment,l),w(_.$$.fragment,l),w(R.$$.fragment,l),w(E.$$.fragment,l),w(k.$$.fragment,l),w(Q.$$.fragment,l),w(S.$$.fragment,l),w(v.$$.fragment,l),w($.$$.fragment,l),w(L.$$.fragment,l),w(Ml.$$.fragment,l),w(el.$$.fragment,l),w(yl.$$.fragment,l),w(Tl.$$.fragment,l),w(Il.$$.fragment,l),w(il.$$.fragment,l),w(cl.$$.fragment,l),w(dl.$$.fragment,l),w(rl.$$.fragment,l),w(fl.$$.fragment,l),w(hl.$$.fragment,l),w(Gl.$$.fragment,l),w(Xl.$$.fragment,l),ts=!1},d(l){l&&(s(Wl),s(Ql),s(Sl),s(Nl),s(gl),s(c),s(Yl),s(d),s(Hl),s(A),s(vl),s(xl),s(b),s(zl),s($l),s(u),s(Ll),s(Dl),s(Z),s(ql),s(B),s(Pl),s(Kl),s(G),s(Ol),s(lM),s(MM),s(sM),s(X),s(eM),s(V),s(tM),s(aM),s(F),s(nM),s(W),s(yM),s(UM),s(N),s(JM),s(g),s(TM),s(Y),s(jM),s(H),s(pM),s(wM),s(x),s(IM),s(z),s(CM),s(iM),s(mM),s(D),s(oM),s(q),s(cM),s(P),s(dM),s(K),s(AM),s(O),s(rM),s(ll),s(bM),s(fM),s(sl),s(uM),s(hM),s(tl),s(ZM),s(al),s(BM),s(nl),s(_M),s(GM),s(Ul),s(RM),s(Jl),s(EM),s(kM),s(jl),s(XM),s(pl),s(VM),s(wl),s(QM),s(FM),s(Cl),s(WM),s(SM),s(ml),s(NM),s(ol),s(gM),s(YM),s(HM),s(Al),s(vM),s(xM),s(bl),s(zM),s($M),s(ul),s(LM),s(DM),s(Zl),s(qM),s(Bl),s(PM),s(_l),s(KM),s(OM),s(Rl),s(ls),s(El),s(Ms),s(kl),s(ss),s(es),s(Fl)),s(i),I(m,l),I(o,l),I(r,l),I(f,l),I(h,l),I(_,l),I(R,l),I(E,l),I(k,l),I(Q,l),I(S,l),I(v,l),I($,l),I(L,l),I(Ml,l),I(el,l),I(yl,l),I(Tl,l),I(Il,l),I(il,l),I(cl,l),I(dl,l),I(rl,l),I(fl,l),I(hl,l),I(Gl,l),I(Xl,l)}}}const Je='{"title":"在单个 GPU 上针对自定义代码微调代码 LLM","local":"在单个-gpu-上针对自定义代码微调代码-llm","sections":[{"title":"数据集","local":"数据集","sections":[],"depth":2},{"title":"模型","local":"模型","sections":[],"depth":2},{"title":"准备数据","local":"准备数据","sections":[],"depth":2},{"title":"准备模型","local":"准备模型","sections":[],"depth":2},{"title":"训练模型","local":"训练模型","sections":[],"depth":2},{"title":"推理","local":"推理","sections":[],"depth":2}],"depth":1}';function Te(as){return Me(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ce extends se{constructor(i){super(),ee(this,i,Te,Ue,Os,{})}}export{Ce as component};
