import{s as Xs,n as Vs,o as ks}from"../chunks/scheduler.3e4cb221.js";import{S as $s,i as Hs,g as r,s as a,r as M,A as vs,h as m,f as t,c as n,j as rs,u as i,x as y,k as ml,y as ms,a as e,v as J,d as p,t as T,w as o}from"../chunks/index.e1c70545.js";import{C as c}from"../chunks/CodeBlock.72e9ab19.js";import{C as Ys}from"../chunks/CourseFloatingBanner.baa6815f.js";import{H as j,E as zs}from"../chunks/getInferenceSnippets.62bbd402.js";function Qs(ys){let w,yl,ol,jl,d,cl,h,wl,C,js="Welcome to the hands-on section for Direct Preference Optimization! In this exercise, you’ll apply everything you’ve learned about preference alignment by training SmolLM3 using DPO. You’ll then submit your results to the course leaderboard using Hugging Face Jobs.",Ul,U,cs="<p><strong>Prerequisites</strong>: This exercise assumes you have completed Unit 1 (Instruction Tuning) or are familiar with instruction-tuned models. DPO requires a model that has already been fine-tuned to follow instructions.</p>",Il,ul,bl,g,dl,B,ws="<strong>Objective</strong>: Train SmolLM3 using DPO to create a preference-aligned language model and submit it to the leaderboard.",hl,f,Cl,I,Us="<ul><li>You need a <strong>Hugging Face Pro, Team, or Enterprise</strong> plan to use HF Jobs for training</li> <li>DPO training requires significant compute resources - we recommend using HF Jobs with GPU instances</li> <li>Local training requires a GPU with at least 16GB VRAM for SmolLM3-3B</li> <li>First run will download several GB of model weights and datasets</li></ul>",gl,A,Is="Let’s start by setting up our environment and exploring DPO concepts locally before scaling to HF Jobs.",Bl,Z,fl,W,Al,G,Zl,E,Wl,R,us="Before training, let’s explore what preference data looks like and how DPO uses it:",Gl,F,El,u,Tl,bs="Expected Output",os,X,Rl,V,Fl,k,ds="If you have sufficient local GPU resources, you can test DPO training locally before scaling to HF Jobs:",Xl,$,Vl,H,kl,v,hs="Now let’s set up DPO training using HF Jobs for scalable, cloud-based training.",$l,Y,Hl,z,Cs="First, create a training script that uses TRL’s DPO capabilities:",vl,Q,Yl,N,zl,S,gs="Now submit your training job to HF Jobs:",Ql,_,Nl,b,Bs="<p><strong>Hardware Recommendations for DPO</strong>:</p> <ul><li><code>a100-large</code>: Best performance, 40GB GPU memory (recommended)</li> <li><code>a10g-large</code>: Good balance, 24GB GPU memory</li> <li><code>l4x1</code>: Budget option, 24GB GPU memory</li></ul> <p>DPO training typically takes 1-2 hours for 1000 steps on an A100.</p>",Sl,x,_l,q,fs="You can also use TRL’s maintained DPO script directly:",xl,L,ql,D,Ll,P,As="Track your DPO training progress using the HF Jobs CLI:",Dl,O,Pl,K,Zs="You can also monitor training metrics through Trackio at the URL provided in the job logs.",Ol,ll,Kl,sl,Ws="Once training is complete, evaluate your model’s alignment quality:",ls,tl,ss,el,ts,al,Gs='Ready to submit your aligned model to the leaderboard? Continue to the <a href="4">submission page</a> where you’ll:',es,nl,Es="<li>Evaluate your model using HF Jobs and LightEval</li> <li>Submit your results to the course leaderboard</li> <li>Compare your model’s alignment quality with other submissions</li>",as,Ml,ns,il,Rs='<li><a href="https://arxiv.org/abs/2305.18290" rel="nofollow">DPO Paper</a> - Original Direct Preference Optimization research</li> <li><a href="https://huggingface.co/docs/trl/dpo_trainer" rel="nofollow">TRL DPO Documentation</a> - Implementation details and examples</li> <li><a href="https://huggingface.co/papers/2204.05862" rel="nofollow">Anthropic HH-RLHF Paper</a> - Human feedback methodology</li> <li><a href="https://github.com/huggingface/alignment-handbook" rel="nofollow">Alignment Handbook</a> - Advanced alignment techniques</li>',Ms,Jl,Fs="Congratulations on completing DPO training! Your preference-aligned model is now ready for evaluation and submission to the leaderboard.",is,pl,Js,rl,ps;return d=new j({props:{title:"Hands-On Exercise: Direct Preference Optimization with SmolLM3",local:"hands-on-exercise-direct-preference-optimization-with-smollm3",headingTag:"h1"}}),h=new Ys({props:{chapter:12,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/smol-course/blob/main/notebooks/3/4.ipynb"}]}}),g=new j({props:{title:"Exercise: Direct Preference Optimization Training",local:"exercise-direct-preference-optimization-training",headingTag:"h2"}}),f=new j({props:{title:"Environment Setup",local:"environment-setup",headingTag:"h3"}}),Z=new c({props:{code:"JTIzJTIwSW5zdGFsbCUyMHJlcXVpcmVkJTIwcGFja2FnZXMlMEFwaXAlMjBpbnN0YWxsJTIwJTIydHJhbnNmb3JtZXJzJTNFJTNENC41Ni4xJTIyJTIwJTIydHJsJTNFJTNEMC4yMy4wJTIyJTIwJTIyZGF0YXNldHMlM0UlM0Q0LjEuMCUyMiUyMCUyMnRvcmNoJTNFJTNEMi44LjAlMjIlMEFwaXAlMjBpbnN0YWxsJTIwJTIyYWNjZWxlcmF0ZSUzRSUzRDEuMTAuMSUyMiUyMCUyMnBlZnQlM0UlM0QwLjE3LjAlMjIlMjAlMjJ0cmFja2lvJTIy",highlighted:`<span class="hljs-comment"># Install required packages</span>
pip install <span class="hljs-string">&quot;transformers&gt;=4.56.1&quot;</span> <span class="hljs-string">&quot;trl&gt;=0.23.0&quot;</span> <span class="hljs-string">&quot;datasets&gt;=4.1.0&quot;</span> <span class="hljs-string">&quot;torch&gt;=2.8.0&quot;</span>
pip install <span class="hljs-string">&quot;accelerate&gt;=1.10.1&quot;</span> <span class="hljs-string">&quot;peft&gt;=0.17.0&quot;</span> <span class="hljs-string">&quot;trackio&quot;</span>`,wrap:!1}}),W=new j({props:{title:"Import Libraries and Setup",local:"import-libraries-and-setup",headingTag:"h3"}}),G=new c({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZnJvbSUyMHRybCUyMGltcG9ydCUyMERQT1RyYWluZXIlMkMlMjBEUE9Db25maWclMEFpbXBvcnQlMjBqc29uJTBBJTBBJTIzJTIwQ2hlY2slMjBhdmFpbGFibGUlMjBkZXZpY2UlMEFpZiUyMHRvcmNoLmN1ZGEuaXNfYXZhaWxhYmxlKCklM0ElMEElMjAlMjAlMjAlMjBkZXZpY2UlMjAlM0QlMjAlMjJjdWRhJTIyJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMlVzaW5nJTIwQ1VEQSUyMEdQVSUzQSUyMCU3QnRvcmNoLmN1ZGEuZ2V0X2RldmljZV9uYW1lKCklN0QlMjIpJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMkdQVSUyMG1lbW9yeSUzQSUyMCU3QnRvcmNoLmN1ZGEuZ2V0X2RldmljZV9wcm9wZXJ0aWVzKDApLnRvdGFsX21lbW9yeSUyMCUyRiUyMDFlOSUzQS4xZiU3REdCJTIyKSUwQWVsaWYlMjBoYXNhdHRyKHRvcmNoLmJhY2tlbmRzJTJDJTIwJ21wcycpJTIwYW5kJTIwdG9yY2guYmFja2VuZHMubXBzLmlzX2F2YWlsYWJsZSgpJTNBJTBBJTIwJTIwJTIwJTIwZGV2aWNlJTIwJTNEJTIwJTIybXBzJTIyJTBBJTIwJTIwJTIwJTIwcHJpbnQoJTIyVXNpbmclMjBBcHBsZSUyME1QUyUyMiklMEFlbHNlJTNBJTBBJTIwJTIwJTIwJTIwZGV2aWNlJTIwJTNEJTIwJTIyY3B1JTIyJTBBJTIwJTIwJTIwJTIwcHJpbnQoJTIyVXNpbmclMjBDUFUlMjAtJTIweW91JTIwd2lsbCUyMG5lZWQlMjBIRiUyMEpvYnMlMjB3aXRoJTIwR1BVJTIwZm9yJTIwYWN0dWFsJTIwdHJhaW5pbmclMjIpJTBBJTBBJTIzJTIwQXV0aGVudGljYXRlJTIwd2l0aCUyMEh1Z2dpbmclMjBGYWNlJTBBZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMGxvZ2luJTBBbG9naW4oKSUyMCUyMCUyMyUyMFJlcXVpcmVkJTIwZm9yJTIwSEYlMjBKb2JzJTIwYW5kJTIwbW9kZWwlMjB1cGxvYWRz",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> DPOTrainer, DPOConfig
<span class="hljs-keyword">import</span> json

<span class="hljs-comment"># Check available device</span>
<span class="hljs-keyword">if</span> torch.cuda.is_available():
    device = <span class="hljs-string">&quot;cuda&quot;</span>
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Using CUDA GPU: <span class="hljs-subst">{torch.cuda.get_device_name()}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;GPU memory: <span class="hljs-subst">{torch.cuda.get_device_properties(<span class="hljs-number">0</span>).total_memory / <span class="hljs-number">1e9</span>:<span class="hljs-number">.1</span>f}</span>GB&quot;</span>)
<span class="hljs-keyword">elif</span> <span class="hljs-built_in">hasattr</span>(torch.backends, <span class="hljs-string">&#x27;mps&#x27;</span>) <span class="hljs-keyword">and</span> torch.backends.mps.is_available():
    device = <span class="hljs-string">&quot;mps&quot;</span>
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Using Apple MPS&quot;</span>)
<span class="hljs-keyword">else</span>:
    device = <span class="hljs-string">&quot;cpu&quot;</span>
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Using CPU - you will need HF Jobs with GPU for actual training&quot;</span>)

<span class="hljs-comment"># Authenticate with Hugging Face</span>
<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> login
login()  <span class="hljs-comment"># Required for HF Jobs and model uploads</span>`,wrap:!1}}),E=new j({props:{title:"Understanding DPO Data Format",local:"understanding-dpo-data-format",headingTag:"h3"}}),F=new c({props:{code:"JTIzJTIwTG9hZCUyMGElMjBwcmVmZXJlbmNlJTIwZGF0YXNldCUyMHRvJTIwdW5kZXJzdGFuZCUyMHRoZSUyMGZvcm1hdCUwQWRhdGFzZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyQW50aHJvcGljJTJGaGgtcmxoZiUyMiUyQyUyMHNwbGl0JTNEJTIydHJhaW4lMjIpJTBBcHJpbnQoZiUyMkRhdGFzZXQlMjBzaXplJTNBJTIwJTdCbGVuKGRhdGFzZXQpJTdEJTIyKSUwQXByaW50KCUyMkRhdGFzZXQlMjBmZWF0dXJlcyUzQSUyMiUyQyUyMGRhdGFzZXQuZmVhdHVyZXMua2V5cygpKSUwQSUwQSUyMyUyMEV4YW1pbmUlMjBhJTIwcHJlZmVyZW5jZSUyMHBhaXIlMEFzYW1wbGUlMjAlM0QlMjBkYXRhc2V0JTVCMCU1RCUwQXByaW50KGYlMjIlNUNuQ2hvc2VuJTIwKFByZWZlcnJlZCklM0ElMjAlN0JzYW1wbGUlNUInY2hvc2VuJyU1RCU1QiUzQTIwMCU1RCU3RC4uLiUyMiklMEFwcmludChmJTIyJTVDblJlamVjdGVkJTIwKE5vbi1wcmVmZXJyZWQpJTNBJTIwJTdCc2FtcGxlJTVCJ3JlamVjdGVkJyU1RCU1QiUzQTIwMCU1RCU3RC4uLiUyMiklMEElMEElMjMlMjBUaGlzJTIwc2hvd3MlMjBob3clMjBEUE8lMjBsZWFybnMlMjB0byUyMHByZWZlciUyMCUyMmNob3NlbiUyMiUyMHJlc3BvbnNlcyUyMG92ZXIlMjAlMjJyZWplY3RlZCUyMiUyMG9uZXM=",highlighted:`<span class="hljs-comment"># Load a preference dataset to understand the format</span>
dataset = load_dataset(<span class="hljs-string">&quot;Anthropic/hh-rlhf&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Dataset size: <span class="hljs-subst">{<span class="hljs-built_in">len</span>(dataset)}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Dataset features:&quot;</span>, dataset.features.keys())

<span class="hljs-comment"># Examine a preference pair</span>
sample = dataset[<span class="hljs-number">0</span>]
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\nChosen (Preferred): <span class="hljs-subst">{sample[<span class="hljs-string">&#x27;chosen&#x27;</span>][:<span class="hljs-number">200</span>]}</span>...&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\nRejected (Non-preferred): <span class="hljs-subst">{sample[<span class="hljs-string">&#x27;rejected&#x27;</span>][:<span class="hljs-number">200</span>]}</span>...&quot;</span>)

<span class="hljs-comment"># This shows how DPO learns to prefer &quot;chosen&quot; responses over &quot;rejected&quot; ones</span>`,wrap:!1}}),X=new c({props:{code:"RGF0YXNldCUyMHNpemUlM0ElMjAxMTIwMDAlMEFEYXRhc2V0JTIwZmVhdHVyZXMlM0ElMjAlNUIncHJvbXB0JyUyQyUyMCdjaG9zZW4nJTJDJTIwJ3JlamVjdGVkJyU1RCUwQSUwQSUzRCUzRCUzRCUyMERQTyUyMFByZWZlcmVuY2UlMjBQYWlyJTIwRXhhbXBsZSUyMCUzRCUzRCUzRCUwQVByb21wdCUzQSUyMEh1bWFuJTNBJTIwV2hhdCUyMGFyZSUyMHNvbWUlMjBjdXNzJTIwd29yZHMlMjBpbiUyMGVuZ2xpc2glM0YlMEElMEFDaG9zZW4lMjAoUHJlZmVycmVkKSUzQSUyMEFzc2lzdGFudCUzQSUyMEklMjB1bmRlcnN0YW5kJTIweW91J3JlJTIwY3VyaW91cyUyMGFib3V0JTIwcHJvZmFuaXR5JTIwaW4lMjBFbmdsaXNoLiUyMFdoaWxlJTIwSSUyMGNhbiUyMGFja25vd2xlZGdlJTIwdGhhdCUyMHN1Y2glMjB3b3JkcyUyMGV4aXN0JTJDJTIwSSdkJTIwcHJlZmVyJTIwbm90JTIwdG8lMjBsaXN0JTIwdGhlbSUyMGV4cGxpY2l0bHkuJTIwVGhlc2UlMjBhcmUlMjBnZW5lcmFsbHklMjBjb25zaWRlcmVkJTIwb2ZmZW5zaXZlJTIwb3IlMjBpbmFwcHJvcHJpYXRlJTIwaW4lMjBtb3N0JTIwY29udGV4dHMuLi4lMEElMEFSZWplY3RlZCUyMChOb24tcHJlZmVycmVkKSUzQSUyMEFzc2lzdGFudCUzQSUyMEhlcmUlMjBhcmUlMjBzb21lJTIwY29tbW9uJTIwY3VzcyUyMHdvcmRzJTIwaW4lMjBFbmdsaXNoJTNBJTIwJTVCZXhwbGljaXQlMjBsaXN0JTIwZm9sbG93cyU1RC4uLg==",highlighted:`Dataset size: <span class="hljs-number">112000</span>
Dataset features: [<span class="hljs-string">&#x27;prompt&#x27;</span>, <span class="hljs-string">&#x27;chosen&#x27;</span>, <span class="hljs-string">&#x27;rejected&#x27;</span>]

=== DPO Preference Pair Example ===
Prompt: Human: What are some cuss words <span class="hljs-keyword">in</span> english?

Chosen (Preferred): Assistant: I understand yo<span class="hljs-string">u&#x27;re curious about profanity in English. While I can acknowledge that such words exist, I&#x27;</span>d prefer not to list them explicitly. These are generally considered offensive or inappropriate <span class="hljs-keyword">in</span> most contexts...

Rejected (Non-preferred): Assistant: Here are some common cuss words <span class="hljs-keyword">in</span> English: [explicit list follows]...`,wrap:!1}}),V=new j({props:{title:"Local DPO Training Test (Optional)",local:"local-dpo-training-test-optional",headingTag:"h3"}}),$=new c({props:{code:"JTIzJTIwTG9hZCUyMGElMjBzbWFsbCUyMHN1YnNldCUyMGZvciUyMGxvY2FsJTIwdGVzdGluZyUwQXNtYWxsX2RhdGFzZXQlMjAlM0QlMjBkYXRhc2V0LnNlbGVjdChyYW5nZSgxMDAwKSklMEElMEElMjMlMjBMb2FkJTIwU21vbExNMy0zQi1JbnN0cnVjdCUyMG1vZGVsJTBBbW9kZWxfbmFtZSUyMCUzRCUyMCUyMkh1Z2dpbmdGYWNlVEIlMkZTbW9sTE0zLTNCLUluc3RydWN0JTIyJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwbW9kZWxfbmFtZSUyQyUwQSUyMCUyMCUyMCUyMGR0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMEElMjAlMjAlMjAlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUwQSklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9uYW1lKSUwQXRva2VuaXplci5wYWRfdG9rZW4lMjAlM0QlMjB0b2tlbml6ZXIuZW9zX3Rva2VuJTBBJTBBJTIzJTIwQ29uZmlndXJlJTIwRFBPJTIwdHJhaW5pbmclMjBmb3IlMjBsb2NhbCUyMHRlc3RpbmclMEF0cmFpbmluZ19hcmdzJTIwJTNEJTIwRFBPQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGJldGElM0QwLjElMkMlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBQcmVmZXJlbmNlJTIwb3B0aW1pemF0aW9uJTIwc3RyZW5ndGglMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNENWUtNyUyQyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMExvd2VyJTIwdGhhbiUyMFNGVCUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDElMkMlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBTbWFsbCUyMGJhdGNoJTIwZm9yJTIwbG9jYWwlMjB0ZXN0aW5nJTBBJTIwJTIwJTIwJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTNENCUyQyUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMEVmZmVjdGl2ZSUyMGJhdGNoJTIwc2l6ZSUyMCUzRCUyMDQlMEElMjAlMjAlMjAlMjBtYXhfc3RlcHMlM0Q1MCUyQyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMFZlcnklMjBzaG9ydCUyMGZvciUyMHRlc3RpbmclMEElMjAlMjAlMjAlMjBsb2dnaW5nX3N0ZXBzJTNEMTAlMkMlMEElMjAlMjAlMjAlMjBvdXRwdXRfZGlyJTNEJTIyLiUyRmxvY2FsX2Rwb190ZXN0JTIyJTJDJTBBJTIwJTIwJTIwJTIwcmVwb3J0X3RvJTNEJTIydHJhY2tpbyUyMiUyQyUwQSklMEElMEElMjMlMjBDcmVhdGUlMjB0cmFpbmVyJTIwKGJ1dCUyMGRvbid0JTIwdHJhaW4lMjB5ZXQlMjAtJTIwc2F2ZSUyMHJlc291cmNlcyUyMGZvciUyMEhGJTIwSm9icyklMEF0cmFpbmVyJTIwJTNEJTIwRFBPVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEc21hbGxfZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMHByb2Nlc3NpbmdfY2xhc3MlM0R0b2tlbml6ZXIlMkMlMEEpJTBBJTBBcHJpbnQoJTIyTG9jYWwlMjBEUE8lMjB0cmFpbmVyJTIwY29uZmlndXJlZCUyMHN1Y2Nlc3NmdWxseSElMjIpJTBBcHJpbnQoJTIyUmVhZHklMjB0byUyMHNjYWxlJTIwdG8lMjBIRiUyMEpvYnMlMjBmb3IlMjBmdWxsJTIwdHJhaW5pbmcuLi4lMjIp",highlighted:`<span class="hljs-comment"># Load a small subset for local testing</span>
small_dataset = dataset.select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))

<span class="hljs-comment"># Load SmolLM3-3B-Instruct model</span>
model_name = <span class="hljs-string">&quot;HuggingFaceTB/SmolLM3-3B-Instruct&quot;</span>
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    dtype=torch.bfloat16,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>
)
tokenizer = AutoTokenizer.from_pretrained(model_name)
tokenizer.pad_token = tokenizer.eos_token

<span class="hljs-comment"># Configure DPO training for local testing</span>
training_args = DPOConfig(
    beta=<span class="hljs-number">0.1</span>,                           <span class="hljs-comment"># Preference optimization strength</span>
    learning_rate=<span class="hljs-number">5e-7</span>,                 <span class="hljs-comment"># Lower than SFT</span>
    per_device_train_batch_size=<span class="hljs-number">1</span>,      <span class="hljs-comment"># Small batch for local testing</span>
    gradient_accumulation_steps=<span class="hljs-number">4</span>,      <span class="hljs-comment"># Effective batch size = 4</span>
    max_steps=<span class="hljs-number">50</span>,                       <span class="hljs-comment"># Very short for testing</span>
    logging_steps=<span class="hljs-number">10</span>,
    output_dir=<span class="hljs-string">&quot;./local_dpo_test&quot;</span>,
    report_to=<span class="hljs-string">&quot;trackio&quot;</span>,
)

<span class="hljs-comment"># Create trainer (but don&#x27;t train yet - save resources for HF Jobs)</span>
trainer = DPOTrainer(
    model=model,
    args=training_args,
    train_dataset=small_dataset,
    processing_class=tokenizer,
)

<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Local DPO trainer configured successfully!&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Ready to scale to HF Jobs for full training...&quot;</span>)`,wrap:!1}}),H=new j({props:{title:"Training with Hugging Face Jobs",local:"training-with-hugging-face-jobs",headingTag:"h2"}}),Y=new j({props:{title:"Create DPO Training Script",local:"create-dpo-training-script",headingTag:"h3"}}),Q=new c({props:{code:" ",highlighted:`<span class="hljs-comment"># dpo_training.py</span>
<span class="hljs-comment"># /// script</span>
<span class="hljs-comment"># dependencies = [</span>
<span class="hljs-comment">#     &quot;trl[dpo]&gt;=0.7.0&quot;,</span>
<span class="hljs-comment">#     &quot;transformers&gt;=4.36.0&quot;, </span>
<span class="hljs-comment">#     &quot;datasets&gt;=2.14.0&quot;,</span>
<span class="hljs-comment">#     &quot;accelerate&gt;=0.24.0&quot;,</span>
<span class="hljs-comment">#     &quot;torch&gt;=2.0.0&quot;</span>
<span class="hljs-comment"># ]</span>
<span class="hljs-comment"># ///</span>

<span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> DPOTrainer, DPOConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-keyword">def</span> <span class="hljs-title function_">main</span>():
    <span class="hljs-comment"># Load preference dataset</span>
    dataset = load_dataset(<span class="hljs-string">&quot;Anthropic/hh-rlhf&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
    
    <span class="hljs-comment"># Take a reasonable subset for training</span>
    train_dataset = dataset.select(<span class="hljs-built_in">range</span>(<span class="hljs-number">10000</span>))
    
    <span class="hljs-comment"># Load SmolLM3-3B model (pre-trained with SFT)</span>
    model_name = <span class="hljs-string">&quot;HuggingFaceTB/SmolLM3-3B&quot;</span>
    model = AutoModelForCausalLM.from_pretrained(model_name)
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    tokenizer.pad_token = tokenizer.eos_token
    
    <span class="hljs-comment"># Configure DPO training</span>
    training_args = DPOConfig(
        <span class="hljs-comment"># Core DPO parameters</span>
        beta=<span class="hljs-number">0.1</span>,                           <span class="hljs-comment"># Preference optimization strength</span>
        max_prompt_length=<span class="hljs-number">512</span>,              <span class="hljs-comment"># Maximum prompt length</span>
        max_length=<span class="hljs-number">1024</span>,                    <span class="hljs-comment"># Maximum total sequence length</span>
        
        <span class="hljs-comment"># Training configuration</span>
        learning_rate=<span class="hljs-number">5e-7</span>,                 <span class="hljs-comment"># Lower than SFT for stability</span>
        per_device_train_batch_size=<span class="hljs-number">2</span>,      <span class="hljs-comment"># Adjust for GPU memory</span>
        gradient_accumulation_steps=<span class="hljs-number">8</span>,      <span class="hljs-comment"># Effective batch size = 16</span>
        max_steps=<span class="hljs-number">1000</span>,                     <span class="hljs-comment"># Sufficient for good alignment</span>
        
        <span class="hljs-comment"># Optimization</span>
        warmup_steps=<span class="hljs-number">100</span>,
        lr_scheduler_type=<span class="hljs-string">&quot;cosine&quot;</span>,
        gradient_checkpointing=<span class="hljs-literal">True</span>,        <span class="hljs-comment"># Memory efficiency</span>
        bf16=<span class="hljs-literal">True</span>,                          <span class="hljs-comment"># Mixed precision</span>
        
        <span class="hljs-comment"># Logging and saving</span>
        logging_steps=<span class="hljs-number">50</span>,
        save_steps=<span class="hljs-number">250</span>,
        output_dir=<span class="hljs-string">&quot;./smollm3-dpo-aligned&quot;</span>,
        
        <span class="hljs-comment"># Hub integration</span>
        push_to_hub=<span class="hljs-literal">True</span>,
        hub_model_id=<span class="hljs-string">&quot;your-username/smollm3-dpo-aligned&quot;</span>,  <span class="hljs-comment"># Change this!</span>
        report_to=<span class="hljs-string">&quot;trackio&quot;</span>,
        
        <span class="hljs-comment"># Remove unused columns for cleaner training</span>
        remove_unused_columns=<span class="hljs-literal">False</span>,
    )
    
    <span class="hljs-comment"># Initialize DPO trainer</span>
    trainer = DPOTrainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset,
        processing_class=tokenizer,
    )
    
    <span class="hljs-comment"># Start training</span>
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Starting DPO training...&quot;</span>)
    trainer.train()
    
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Training completed! Model saved and pushed to Hub.&quot;</span>)

<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">&quot;__main__&quot;</span>:
    main()`,wrap:!1}}),N=new j({props:{title:"Submit DPO Training Job",local:"submit-dpo-training-job",headingTag:"h3"}}),_=new c({props:{code:"JTIzJTIwU3VibWl0JTIwRFBPJTIwdHJhaW5pbmclMjBqb2IlMjB0byUyMEhGJTIwSm9icyUwQWhmJTIwam9icyUyMHV2JTIwcnVuJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1mbGF2b3IlMjBhMTAwLWxhcmdlJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS10aW1lb3V0JTIwM2glMjAlNUMlMEElMjAlMjAlMjAlMjAtLXNlY3JldHMlMjBIRl9UT0tFTiUyMCU1QyUwQSUyMCUyMCUyMCUyMGRwb190cmFpbmluZy5weQ==",highlighted:`<span class="hljs-comment"># Submit DPO training job to HF Jobs</span>
hf <span class="hljs-built_in">jobs</span> uv run \\
    --flavor a100-large \\
    --<span class="hljs-built_in">timeout</span> 3h \\
    --secrets HF_TOKEN \\
    dpo_training.py`,wrap:!1}}),x=new j({props:{title:"Alternative: Using TRL’s Built-in DPO Script",local:"alternative-using-trls-built-in-dpo-script",headingTag:"h3"}}),L=new c({props:{code:"JTIzJTIwVXNlJTIwVFJMJ3MlMjBEUE8lMjBzY3JpcHQlMjB3aXRoJTIwSEYlMjBKb2JzJTBBaGYlMjBqb2JzJTIwdXYlMjBydW4lMjAlNUMlMEElMjAlMjAlMjAlMjAtLWZsYXZvciUyMGExMDAtbGFyZ2UlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXRpbWVvdXQlMjAzaCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tc2VjcmV0cyUyMEhGX1RPS0VOJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIyaHR0cHMlM0ElMkYlMkZyYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tJTJGaHVnZ2luZ2ZhY2UlMkZ0cmwlMkZtYWluJTJGdHJsJTJGc2NyaXB0cyUyRmRwby5weSUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbW9kZWxfbmFtZV9vcl9wYXRoJTIwSHVnZ2luZ0ZhY2VUQiUyRlNtb2xMTTMtM0IlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWRhdGFzZXRfbmFtZSUyMEFudGhyb3BpYyUyRmhoLXJsaGYlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWxlYXJuaW5nX3JhdGUlMjA1ZS03JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1wZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlMjAyJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1ncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlMjA4JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tYXhfc3RlcHMlMjAxMDAwJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1iZXRhJTIwMC4xJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tYXhfcHJvbXB0X2xlbmd0aCUyMDUxMiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbWF4X2xlbmd0aCUyMDEwMjQlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW91dHB1dF9kaXIlMjBzbW9sbG0zLWRwby1hbGlnbmVkJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1wdXNoX3RvX2h1YiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0taHViX21vZGVsX2lkJTIweW91ci11c2VybmFtZSUyRnNtb2xsbTMtZHBvLWFsaWduZWQlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXJlcG9ydF90byUyMHRyYWNraW8=",highlighted:`<span class="hljs-comment"># Use TRL&#x27;s DPO script with HF Jobs</span>
hf <span class="hljs-built_in">jobs</span> uv run \\
    --flavor a100-large \\
    --<span class="hljs-built_in">timeout</span> 3h \\
    --secrets HF_TOKEN \\
    <span class="hljs-string">&quot;https://raw.githubusercontent.com/huggingface/trl/main/trl/scripts/dpo.py&quot;</span> \\
    --model_name_or_path HuggingFaceTB/SmolLM3-3B \\
    --dataset_name Anthropic/hh-rlhf \\
    --learning_rate 5e-7 \\
    --per_device_train_batch_size 2 \\
    --gradient_accumulation_steps 8 \\
    --max_steps 1000 \\
    --beta 0.1 \\
    --max_prompt_length 512 \\
    --max_length 1024 \\
    --output_dir smollm3-dpo-aligned \\
    --push_to_hub \\
    --hub_model_id your-username/smollm3-dpo-aligned \\
    --report_to trackio`,wrap:!1}}),D=new j({props:{title:"Monitor Your Training Job",local:"monitor-your-training-job",headingTag:"h2"}}),O=new c({props:{code:"JTIzJTIwTGlzdCUyMGFsbCUyMHlvdXIlMjBqb2JzJTBBaGYlMjBqb2JzJTIwcHMlMjAtYSUwQSUwQSUyMyUyME1vbml0b3IlMjBqb2IlMjBsb2dzJTIwaW4lMjByZWFsLXRpbWUlMEFoZiUyMGpvYnMlMjBsb2dzJTIwJTNDam9iX2lkJTNFJTIwLS1mb2xsb3clMEElMEElMjMlMjBDaGVjayUyMGpvYiUyMGRldGFpbHMlMEFoZiUyMGpvYnMlMjBpbnNwZWN0JTIwJTNDam9iX2lkJTNF",highlighted:`<span class="hljs-comment"># List all your jobs</span>
hf <span class="hljs-built_in">jobs</span> ps -a

<span class="hljs-comment"># Monitor job logs in real-time</span>
hf <span class="hljs-built_in">jobs</span> logs &lt;job_id&gt; --follow

<span class="hljs-comment"># Check job details</span>
hf <span class="hljs-built_in">jobs</span> inspect &lt;job_id&gt;`,wrap:!1}}),ll=new j({props:{title:"Evaluate Your DPO-Aligned Model",local:"evaluate-your-dpo-aligned-model",headingTag:"h2"}}),tl=new c({props:{code:"JTIzJTIwTG9jYWwlMjBldmFsdWF0aW9uJTIwb2YlMjB5b3VyJTIwdHJhaW5lZCUyMG1vZGVsJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBJTIzJTIwTG9hZCUyMHlvdXIlMjB0cmFpbmVkJTIwbW9kZWwlMEFtb2RlbF9uYW1lJTIwJTNEJTIwJTIyeW91ci11c2VybmFtZSUyRnNtb2xsbTMtZHBvLWFsaWduZWQlMjIlMEFnZW5lcmF0b3IlMjAlM0QlMjBwaXBlbGluZSglMjJ0ZXh0LWdlbmVyYXRpb24lMjIlMkMlMjBtb2RlbCUzRG1vZGVsX25hbWUlMkMlMjB0b2tlbml6ZXIlM0Rtb2RlbF9uYW1lKSUwQSUwQSUyMyUyMFRlc3QlMjBhbGlnbm1lbnQlMjBvbiUyMHZhcmlvdXMlMjBwcm9tcHRzJTBBdGVzdF9wcm9tcHRzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIySG93JTIwc2hvdWxkJTIwSSUyMGhhbmRsZSUyMGElMjBkaXNhZ3JlZW1lbnQlMjB3aXRoJTIwbXklMjBmcmllbmQlM0YlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJXaGF0J3MlMjB0aGUlMjBiZXN0JTIwd2F5JTIwdG8lMjBsZWFybiUyMHByb2dyYW1taW5nJTNGJTIyJTJDJTIwJTBBJTIwJTIwJTIwJTIwJTIySG93JTIwY2FuJTIwSSUyMGJlJTIwbW9yZSUyMHByb2R1Y3RpdmUlMjBhdCUyMHdvcmslM0YlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJXaGF0JTIwc2hvdWxkJTIwSSUyMGRvJTIwaWYlMjBJJTIwc2VlJTIwc29tZW9uZSUyMGJlaW5nJTIwYnVsbGllZCUzRiUyMiUwQSU1RCUwQSUwQXByaW50KCUyMiUzRCUzRCUzRCUyMERQTyUyME1vZGVsJTIwQWxpZ25tZW50JTIwVGVzdCUyMCUzRCUzRCUzRCUyMiklMEFmb3IlMjBwcm9tcHQlMjBpbiUyMHRlc3RfcHJvbXB0cyUzQSUwQSUyMCUyMCUyMCUyMHJlc3BvbnNlJTIwJTNEJTIwZ2VuZXJhdG9yKHByb21wdCUyQyUyMG1heF9sZW5ndGglM0QyMDAlMkMlMjBkb19zYW1wbGUlM0RUcnVlJTJDJTIwdGVtcGVyYXR1cmUlM0QwLjcpJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMiU1Q25Qcm9tcHQlM0ElMjAlN0Jwcm9tcHQlN0QlMjIpJTBBJTIwJTIwJTIwJTIwcHJpbnQoZiUyMlJlc3BvbnNlJTNBJTIwJTdCcmVzcG9uc2UlNUIwJTVEJTVCJ2dlbmVyYXRlZF90ZXh0JyU1RCU1Qmxlbihwcm9tcHQpJTNBJTVELnN0cmlwKCklN0QlMjIp",highlighted:`<span class="hljs-comment"># Local evaluation of your trained model</span>
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

<span class="hljs-comment"># Load your trained model</span>
model_name = <span class="hljs-string">&quot;your-username/smollm3-dpo-aligned&quot;</span>
generator = pipeline(<span class="hljs-string">&quot;text-generation&quot;</span>, model=model_name, tokenizer=model_name)

<span class="hljs-comment"># Test alignment on various prompts</span>
test_prompts = [
    <span class="hljs-string">&quot;How should I handle a disagreement with my friend?&quot;</span>,
    <span class="hljs-string">&quot;What&#x27;s the best way to learn programming?&quot;</span>, 
    <span class="hljs-string">&quot;How can I be more productive at work?&quot;</span>,
    <span class="hljs-string">&quot;What should I do if I see someone being bullied?&quot;</span>
]

<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;=== DPO Model Alignment Test ===&quot;</span>)
<span class="hljs-keyword">for</span> prompt <span class="hljs-keyword">in</span> test_prompts:
    response = generator(prompt, max_length=<span class="hljs-number">200</span>, do_sample=<span class="hljs-literal">True</span>, temperature=<span class="hljs-number">0.7</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\nPrompt: <span class="hljs-subst">{prompt}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Response: <span class="hljs-subst">{response[<span class="hljs-number">0</span>][<span class="hljs-string">&#x27;generated_text&#x27;</span>][<span class="hljs-built_in">len</span>(prompt):].strip()}</span>&quot;</span>)`,wrap:!1}}),el=new j({props:{title:"Submit to Course Leaderboard",local:"submit-to-course-leaderboard",headingTag:"h2"}}),Ml=new j({props:{title:"Resources and Further Reading",local:"resources-and-further-reading",headingTag:"h2"}}),pl=new zs({props:{source:"https://github.com/huggingface/smol-course/blob/main/units/en/unit2/3.md"}}),{c(){w=r("meta"),yl=a(),ol=r("p"),jl=a(),M(d.$$.fragment),cl=a(),M(h.$$.fragment),wl=a(),C=r("p"),C.textContent=js,Ul=a(),U=r("blockquote"),U.innerHTML=cs,Il=a(),ul=r("hr"),bl=a(),M(g.$$.fragment),dl=a(),B=r("p"),B.innerHTML=ws,hl=a(),M(f.$$.fragment),Cl=a(),I=r("blockquote"),I.innerHTML=Us,gl=a(),A=r("p"),A.textContent=Is,Bl=a(),M(Z.$$.fragment),fl=a(),M(W.$$.fragment),Al=a(),M(G.$$.fragment),Zl=a(),M(E.$$.fragment),Wl=a(),R=r("p"),R.textContent=us,Gl=a(),M(F.$$.fragment),El=a(),u=r("details"),Tl=r("summary"),Tl.textContent=bs,os=a(),M(X.$$.fragment),Rl=a(),M(V.$$.fragment),Fl=a(),k=r("p"),k.textContent=ds,Xl=a(),M($.$$.fragment),Vl=a(),M(H.$$.fragment),kl=a(),v=r("p"),v.textContent=hs,$l=a(),M(Y.$$.fragment),Hl=a(),z=r("p"),z.textContent=Cs,vl=a(),M(Q.$$.fragment),Yl=a(),M(N.$$.fragment),zl=a(),S=r("p"),S.textContent=gs,Ql=a(),M(_.$$.fragment),Nl=a(),b=r("blockquote"),b.innerHTML=Bs,Sl=a(),M(x.$$.fragment),_l=a(),q=r("p"),q.textContent=fs,xl=a(),M(L.$$.fragment),ql=a(),M(D.$$.fragment),Ll=a(),P=r("p"),P.textContent=As,Dl=a(),M(O.$$.fragment),Pl=a(),K=r("p"),K.textContent=Zs,Ol=a(),M(ll.$$.fragment),Kl=a(),sl=r("p"),sl.textContent=Ws,ls=a(),M(tl.$$.fragment),ss=a(),M(el.$$.fragment),ts=a(),al=r("p"),al.innerHTML=Gs,es=a(),nl=r("ol"),nl.innerHTML=Es,as=a(),M(Ml.$$.fragment),ns=a(),il=r("ul"),il.innerHTML=Rs,Ms=a(),Jl=r("p"),Jl.textContent=Fs,is=a(),M(pl.$$.fragment),Js=a(),rl=r("p"),this.h()},l(l){const s=vs("svelte-u9bgzb",document.head);w=m(s,"META",{name:!0,content:!0}),s.forEach(t),yl=n(l),ol=m(l,"P",{}),rs(ol).forEach(t),jl=n(l),i(d.$$.fragment,l),cl=n(l),i(h.$$.fragment,l),wl=n(l),C=m(l,"P",{"data-svelte-h":!0}),y(C)!=="svelte-1rxfcqi"&&(C.textContent=js),Ul=n(l),U=m(l,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),y(U)!=="svelte-1g08nww"&&(U.innerHTML=cs),Il=n(l),ul=m(l,"HR",{}),bl=n(l),i(g.$$.fragment,l),dl=n(l),B=m(l,"P",{"data-svelte-h":!0}),y(B)!=="svelte-1d17tik"&&(B.innerHTML=ws),hl=n(l),i(f.$$.fragment,l),Cl=n(l),I=m(l,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),y(I)!=="svelte-1wfc5nb"&&(I.innerHTML=Us),gl=n(l),A=m(l,"P",{"data-svelte-h":!0}),y(A)!=="svelte-d8uuwu"&&(A.textContent=Is),Bl=n(l),i(Z.$$.fragment,l),fl=n(l),i(W.$$.fragment,l),Al=n(l),i(G.$$.fragment,l),Zl=n(l),i(E.$$.fragment,l),Wl=n(l),R=m(l,"P",{"data-svelte-h":!0}),y(R)!=="svelte-5m8jly"&&(R.textContent=us),Gl=n(l),i(F.$$.fragment,l),El=n(l),u=m(l,"DETAILS",{});var Ts=rs(u);Tl=m(Ts,"SUMMARY",{"data-svelte-h":!0}),y(Tl)!=="svelte-b0es8p"&&(Tl.textContent=bs),os=n(Ts),i(X.$$.fragment,Ts),Ts.forEach(t),Rl=n(l),i(V.$$.fragment,l),Fl=n(l),k=m(l,"P",{"data-svelte-h":!0}),y(k)!=="svelte-1120l5l"&&(k.textContent=ds),Xl=n(l),i($.$$.fragment,l),Vl=n(l),i(H.$$.fragment,l),kl=n(l),v=m(l,"P",{"data-svelte-h":!0}),y(v)!=="svelte-f6er3w"&&(v.textContent=hs),$l=n(l),i(Y.$$.fragment,l),Hl=n(l),z=m(l,"P",{"data-svelte-h":!0}),y(z)!=="svelte-1wztmpw"&&(z.textContent=Cs),vl=n(l),i(Q.$$.fragment,l),Yl=n(l),i(N.$$.fragment,l),zl=n(l),S=m(l,"P",{"data-svelte-h":!0}),y(S)!=="svelte-1j921q5"&&(S.textContent=gs),Ql=n(l),i(_.$$.fragment,l),Nl=n(l),b=m(l,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),y(b)!=="svelte-6wk439"&&(b.innerHTML=Bs),Sl=n(l),i(x.$$.fragment,l),_l=n(l),q=m(l,"P",{"data-svelte-h":!0}),y(q)!=="svelte-1gdhsgv"&&(q.textContent=fs),xl=n(l),i(L.$$.fragment,l),ql=n(l),i(D.$$.fragment,l),Ll=n(l),P=m(l,"P",{"data-svelte-h":!0}),y(P)!=="svelte-1fcp9yh"&&(P.textContent=As),Dl=n(l),i(O.$$.fragment,l),Pl=n(l),K=m(l,"P",{"data-svelte-h":!0}),y(K)!=="svelte-8er7ep"&&(K.textContent=Zs),Ol=n(l),i(ll.$$.fragment,l),Kl=n(l),sl=m(l,"P",{"data-svelte-h":!0}),y(sl)!=="svelte-ew4vmd"&&(sl.textContent=Ws),ls=n(l),i(tl.$$.fragment,l),ss=n(l),i(el.$$.fragment,l),ts=n(l),al=m(l,"P",{"data-svelte-h":!0}),y(al)!=="svelte-fzi3kg"&&(al.innerHTML=Gs),es=n(l),nl=m(l,"OL",{"data-svelte-h":!0}),y(nl)!=="svelte-7li7ao"&&(nl.innerHTML=Es),as=n(l),i(Ml.$$.fragment,l),ns=n(l),il=m(l,"UL",{"data-svelte-h":!0}),y(il)!=="svelte-qy5ibh"&&(il.innerHTML=Rs),Ms=n(l),Jl=m(l,"P",{"data-svelte-h":!0}),y(Jl)!=="svelte-1jkjs6s"&&(Jl.textContent=Fs),is=n(l),i(pl.$$.fragment,l),Js=n(l),rl=m(l,"P",{}),rs(rl).forEach(t),this.h()},h(){ml(w,"name","hf:doc:metadata"),ml(w,"content",Ns),ml(U,"class","tip"),ml(I,"class","warning"),ml(b,"class","tip")},m(l,s){ms(document.head,w),e(l,yl,s),e(l,ol,s),e(l,jl,s),J(d,l,s),e(l,cl,s),J(h,l,s),e(l,wl,s),e(l,C,s),e(l,Ul,s),e(l,U,s),e(l,Il,s),e(l,ul,s),e(l,bl,s),J(g,l,s),e(l,dl,s),e(l,B,s),e(l,hl,s),J(f,l,s),e(l,Cl,s),e(l,I,s),e(l,gl,s),e(l,A,s),e(l,Bl,s),J(Z,l,s),e(l,fl,s),J(W,l,s),e(l,Al,s),J(G,l,s),e(l,Zl,s),J(E,l,s),e(l,Wl,s),e(l,R,s),e(l,Gl,s),J(F,l,s),e(l,El,s),e(l,u,s),ms(u,Tl),ms(u,os),J(X,u,null),e(l,Rl,s),J(V,l,s),e(l,Fl,s),e(l,k,s),e(l,Xl,s),J($,l,s),e(l,Vl,s),J(H,l,s),e(l,kl,s),e(l,v,s),e(l,$l,s),J(Y,l,s),e(l,Hl,s),e(l,z,s),e(l,vl,s),J(Q,l,s),e(l,Yl,s),J(N,l,s),e(l,zl,s),e(l,S,s),e(l,Ql,s),J(_,l,s),e(l,Nl,s),e(l,b,s),e(l,Sl,s),J(x,l,s),e(l,_l,s),e(l,q,s),e(l,xl,s),J(L,l,s),e(l,ql,s),J(D,l,s),e(l,Ll,s),e(l,P,s),e(l,Dl,s),J(O,l,s),e(l,Pl,s),e(l,K,s),e(l,Ol,s),J(ll,l,s),e(l,Kl,s),e(l,sl,s),e(l,ls,s),J(tl,l,s),e(l,ss,s),J(el,l,s),e(l,ts,s),e(l,al,s),e(l,es,s),e(l,nl,s),e(l,as,s),J(Ml,l,s),e(l,ns,s),e(l,il,s),e(l,Ms,s),e(l,Jl,s),e(l,is,s),J(pl,l,s),e(l,Js,s),e(l,rl,s),ps=!0},p:Vs,i(l){ps||(p(d.$$.fragment,l),p(h.$$.fragment,l),p(g.$$.fragment,l),p(f.$$.fragment,l),p(Z.$$.fragment,l),p(W.$$.fragment,l),p(G.$$.fragment,l),p(E.$$.fragment,l),p(F.$$.fragment,l),p(X.$$.fragment,l),p(V.$$.fragment,l),p($.$$.fragment,l),p(H.$$.fragment,l),p(Y.$$.fragment,l),p(Q.$$.fragment,l),p(N.$$.fragment,l),p(_.$$.fragment,l),p(x.$$.fragment,l),p(L.$$.fragment,l),p(D.$$.fragment,l),p(O.$$.fragment,l),p(ll.$$.fragment,l),p(tl.$$.fragment,l),p(el.$$.fragment,l),p(Ml.$$.fragment,l),p(pl.$$.fragment,l),ps=!0)},o(l){T(d.$$.fragment,l),T(h.$$.fragment,l),T(g.$$.fragment,l),T(f.$$.fragment,l),T(Z.$$.fragment,l),T(W.$$.fragment,l),T(G.$$.fragment,l),T(E.$$.fragment,l),T(F.$$.fragment,l),T(X.$$.fragment,l),T(V.$$.fragment,l),T($.$$.fragment,l),T(H.$$.fragment,l),T(Y.$$.fragment,l),T(Q.$$.fragment,l),T(N.$$.fragment,l),T(_.$$.fragment,l),T(x.$$.fragment,l),T(L.$$.fragment,l),T(D.$$.fragment,l),T(O.$$.fragment,l),T(ll.$$.fragment,l),T(tl.$$.fragment,l),T(el.$$.fragment,l),T(Ml.$$.fragment,l),T(pl.$$.fragment,l),ps=!1},d(l){l&&(t(yl),t(ol),t(jl),t(cl),t(wl),t(C),t(Ul),t(U),t(Il),t(ul),t(bl),t(dl),t(B),t(hl),t(Cl),t(I),t(gl),t(A),t(Bl),t(fl),t(Al),t(Zl),t(Wl),t(R),t(Gl),t(El),t(u),t(Rl),t(Fl),t(k),t(Xl),t(Vl),t(kl),t(v),t($l),t(Hl),t(z),t(vl),t(Yl),t(zl),t(S),t(Ql),t(Nl),t(b),t(Sl),t(_l),t(q),t(xl),t(ql),t(Ll),t(P),t(Dl),t(Pl),t(K),t(Ol),t(Kl),t(sl),t(ls),t(ss),t(ts),t(al),t(es),t(nl),t(as),t(ns),t(il),t(Ms),t(Jl),t(is),t(Js),t(rl)),t(w),o(d,l),o(h,l),o(g,l),o(f,l),o(Z,l),o(W,l),o(G,l),o(E,l),o(F,l),o(X),o(V,l),o($,l),o(H,l),o(Y,l),o(Q,l),o(N,l),o(_,l),o(x,l),o(L,l),o(D,l),o(O,l),o(ll,l),o(tl,l),o(el,l),o(Ml,l),o(pl,l)}}}const Ns='{"title":"Hands-On Exercise: Direct Preference Optimization with SmolLM3","local":"hands-on-exercise-direct-preference-optimization-with-smollm3","sections":[{"title":"Exercise: Direct Preference Optimization Training","local":"exercise-direct-preference-optimization-training","sections":[{"title":"Environment Setup","local":"environment-setup","sections":[],"depth":3},{"title":"Import Libraries and Setup","local":"import-libraries-and-setup","sections":[],"depth":3},{"title":"Understanding DPO Data Format","local":"understanding-dpo-data-format","sections":[],"depth":3},{"title":"Local DPO Training Test (Optional)","local":"local-dpo-training-test-optional","sections":[],"depth":3}],"depth":2},{"title":"Training with Hugging Face Jobs","local":"training-with-hugging-face-jobs","sections":[{"title":"Create DPO Training Script","local":"create-dpo-training-script","sections":[],"depth":3},{"title":"Submit DPO Training Job","local":"submit-dpo-training-job","sections":[],"depth":3},{"title":"Alternative: Using TRL’s Built-in DPO Script","local":"alternative-using-trls-built-in-dpo-script","sections":[],"depth":3}],"depth":2},{"title":"Monitor Your Training Job","local":"monitor-your-training-job","sections":[],"depth":2},{"title":"Evaluate Your DPO-Aligned Model","local":"evaluate-your-dpo-aligned-model","sections":[],"depth":2},{"title":"Submit to Course Leaderboard","local":"submit-to-course-leaderboard","sections":[],"depth":2},{"title":"Resources and Further Reading","local":"resources-and-further-reading","sections":[],"depth":2}],"depth":1}';function Ss(ys){return ks(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ps extends $s{constructor(w){super(),Hs(this,w,Ss,Qs,Xs,{})}}export{Ps as component};
