import{s as z,n as K,o as N}from"../chunks/scheduler.25b97de1.js";import{S as W,i as F,g as i,s as r,r as k,A as J,h as s,f as a,c as o,j as S,u as U,x as b,k as D,y as Q,a as l,v as O,d as j,t as I,w as q}from"../chunks/index.d9030fc9.js";import{H as V,E as X}from"../chunks/EditOnGithub.91d95064.js";function Y(C){let n,x,g,T,p,w,h,R=`There is a growing field of study concerned with investigating the inner working of large-scale transformers like BERT
(that some call “BERTology”). Some good examples of this field are:`,y,f,H=`<li>BERT Rediscovers the Classical NLP Pipeline by Ian Tenney, Dipanjan Das, Ellie Pavlick:
<a href="https://arxiv.org/abs/1905.05950" rel="nofollow">https://arxiv.org/abs/1905.05950</a></li> <li>Are Sixteen Heads Really Better than One? by Paul Michel, Omer Levy, Graham Neubig: <a href="https://arxiv.org/abs/1905.10650" rel="nofollow">https://arxiv.org/abs/1905.10650</a></li> <li>What Does BERT Look At? An Analysis of BERT’s Attention by Kevin Clark, Urvashi Khandelwal, Omer Levy, Christopher D.
Manning: <a href="https://arxiv.org/abs/1906.04341" rel="nofollow">https://arxiv.org/abs/1906.04341</a></li> <li>CAT-probing: A Metric-based Approach to Interpret How Pre-trained Models for Programming Language Attend Code Structure: <a href="https://arxiv.org/abs/2210.04633" rel="nofollow">https://arxiv.org/abs/2210.04633</a></li>`,_,m,A=`In order to help this new field develop, we have included a few additional features in the BERT/GPT/GPT-2 models to
help people access the inner representations, mainly adapted from the great work of Paul Michel
(<a href="https://arxiv.org/abs/1905.10650" rel="nofollow">https://arxiv.org/abs/1905.10650</a>):`,P,u,B=`<li>accessing all the hidden-states of BERT/GPT/GPT-2,</li> <li>accessing all the attention weights for each head of BERT/GPT/GPT-2,</li> <li>retrieving heads output values and gradients to be able to compute head importance score and prune head as explained
in <a href="https://arxiv.org/abs/1905.10650" rel="nofollow">https://arxiv.org/abs/1905.10650</a>.</li>`,E,c,G=`To help you understand and use these features, we have added a specific example script: <a href="https://github.com/huggingface/transformers/tree/main/examples/research_projects/bertology/run_bertology.py" rel="nofollow">bertology.py</a> which extracts information and prune a model pre-trained on
GLUE.`,$,d,L,v,M;return p=new V({props:{title:"BERTology",local:"bertology",headingTag:"h1"}}),d=new X({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/bertology.md"}}),{c(){n=i("meta"),x=r(),g=i("p"),T=r(),k(p.$$.fragment),w=r(),h=i("p"),h.textContent=R,y=r(),f=i("ul"),f.innerHTML=H,_=r(),m=i("p"),m.innerHTML=A,P=r(),u=i("ul"),u.innerHTML=B,E=r(),c=i("p"),c.innerHTML=G,$=r(),k(d.$$.fragment),L=r(),v=i("p"),this.h()},l(e){const t=J("svelte-u9bgzb",document.head);n=s(t,"META",{name:!0,content:!0}),t.forEach(a),x=o(e),g=s(e,"P",{}),S(g).forEach(a),T=o(e),U(p.$$.fragment,e),w=o(e),h=s(e,"P",{"data-svelte-h":!0}),b(h)!=="svelte-e5vxoh"&&(h.textContent=R),y=o(e),f=s(e,"UL",{"data-svelte-h":!0}),b(f)!=="svelte-1e34lw"&&(f.innerHTML=H),_=o(e),m=s(e,"P",{"data-svelte-h":!0}),b(m)!=="svelte-1bavv9a"&&(m.innerHTML=A),P=o(e),u=s(e,"UL",{"data-svelte-h":!0}),b(u)!=="svelte-14erpmf"&&(u.innerHTML=B),E=o(e),c=s(e,"P",{"data-svelte-h":!0}),b(c)!=="svelte-2lqgzt"&&(c.innerHTML=G),$=o(e),U(d.$$.fragment,e),L=o(e),v=s(e,"P",{}),S(v).forEach(a),this.h()},h(){D(n,"name","hf:doc:metadata"),D(n,"content",Z)},m(e,t){Q(document.head,n),l(e,x,t),l(e,g,t),l(e,T,t),O(p,e,t),l(e,w,t),l(e,h,t),l(e,y,t),l(e,f,t),l(e,_,t),l(e,m,t),l(e,P,t),l(e,u,t),l(e,E,t),l(e,c,t),l(e,$,t),O(d,e,t),l(e,L,t),l(e,v,t),M=!0},p:K,i(e){M||(j(p.$$.fragment,e),j(d.$$.fragment,e),M=!0)},o(e){I(p.$$.fragment,e),I(d.$$.fragment,e),M=!1},d(e){e&&(a(x),a(g),a(T),a(w),a(h),a(y),a(f),a(_),a(m),a(P),a(u),a(E),a(c),a($),a(L),a(v)),a(n),q(p,e),q(d,e)}}}const Z='{"title":"BERTology","local":"bertology","sections":[],"depth":1}';function ee(C){return N(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ne extends W{constructor(n){super(),F(this,n,ee,Y,z,{})}}export{ne as component};
