import{s as gt,o as _t,n as Fe}from"../chunks/scheduler.25b97de1.js";import{S as Mt,i as bt,g as d,s,r as f,A as vt,h as l,f as n,c as a,j as P,u as h,x as m,k as q,y as r,a as c,v as g,d as _,t as M,w as b}from"../chunks/index.d9030fc9.js";import{T as Qe}from"../chunks/Tip.baa67368.js";import{D as te}from"../chunks/Docstring.e257edda.js";import{C as Tt}from"../chunks/CodeBlock.e6cd0d95.js";import{E as wt}from"../chunks/ExampleCodeBlock.20db4b6e.js";import{H as Xe,E as $t}from"../chunks/EditOnGithub.91d95064.js";function yt(k){let o,p;return o=new Tt({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEpldE1vZU1vZGVsJTJDJTIwSmV0TW9lQ29uZmlnJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMEpldE1vZSUyMDRCJTIwc3R5bGUlMjBjb25maWd1cmF0aW9uJTBBY29uZmlndXJhdGlvbiUyMCUzRCUyMEpldE1vZUNvbmZpZygpJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMG1vZGVsJTIwZnJvbSUyMHRoZSUyMEpldE1vZSUyMDRCJTIwc3R5bGUlMjBjb25maWd1cmF0aW9uJTBBbW9kZWwlMjAlM0QlMjBKZXRNb2VNb2RlbChjb25maWd1cmF0aW9uKSUwQSUwQSUyMyUyMEFjY2Vzc2luZyUyMHRoZSUyMG1vZGVsJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBtb2RlbC5jb25maWc=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> JetMoeModel, JetMoeConfig

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a JetMoe 4B style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = JetMoeConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a model from the JetMoe 4B style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = JetMoeModel(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`,wrap:!1}}),{c(){f(o.$$.fragment)},l(i){h(o.$$.fragment,i)},m(i,$){g(o,i,$),p=!0},p:Fe,i(i){p||(_(o.$$.fragment,i),p=!0)},o(i){M(o.$$.fragment,i),p=!1},d(i){b(o,i)}}}function xt(k){let o,p=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){o=d("p"),o.innerHTML=p},l(i){o=l(i,"P",{"data-svelte-h":!0}),m(o)!=="svelte-fincs2"&&(o.innerHTML=p)},m(i,$){c(i,o,$)},p:Fe,d(i){i&&n(o)}}}function kt(k){let o,p=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){o=d("p"),o.innerHTML=p},l(i){o=l(i,"P",{"data-svelte-h":!0}),m(o)!=="svelte-fincs2"&&(o.innerHTML=p)},m(i,$){c(i,o,$)},p:Fe,d(i){i&&n(o)}}}function Jt(k){let o,p=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){o=d("p"),o.innerHTML=p},l(i){o=l(i,"P",{"data-svelte-h":!0}),m(o)!=="svelte-fincs2"&&(o.innerHTML=p)},m(i,$){c(i,o,$)},p:Fe,d(i){i&&n(o)}}}function Ct(k){let o,p,i,$,W,_e,H,Me,N,Ke=`<strong>JetMoe-8B</strong> is an 8B Mixture-of-Experts (MoE) language model developed by <a href="https://scholar.google.com.hk/citations?user=qff5rRYAAAAJ" rel="nofollow">Yikang Shen</a> and <a href="https://myshell.ai/" rel="nofollow">MyShell</a>.
JetMoe project aims to provide a LLaMA2-level performance and efficient language model with a limited budget.
To achieve this goal, JetMoe uses a sparsely activated architecture inspired by the <a href="https://arxiv.org/abs/2306.04640" rel="nofollow">ModuleFormer</a>.
Each JetMoe block consists of two MoE layers: Mixture of Attention Heads and Mixture of MLP Experts.
Given the input tokens, it activates a subset of its experts to process them.
This sparse activation schema enables JetMoe to achieve much better training throughput than similar size dense models.
The training throughput of JetMoe-8B is around 100B tokens per day on a cluster of 96 H100 GPUs with a straightforward 3-way pipeline parallelism strategy.`,be,E,et='This model was contributed by <a href="https://huggingface.co/YikangS" rel="nofollow">Yikang Shen</a>.',ve,D,Te,v,B,Pe,oe,tt=`This is the configuration class to store the configuration of a <a href="/docs/transformers/pr_30621/en/model_doc/jetmoe#transformers.JetMoeModel">JetMoeModel</a>. It is used to instantiate a
JetMoe model according to the specified arguments, defining the model architecture. Instantiating a configuration
with the defaults will yield a configuration of the JetMoe-4B.`,qe,ne,ot='<a href="https://huggingface.co/jetmoe/jetmoe-8b" rel="nofollow">jetmoe/jetmoe-8b</a>',je,re,nt=`Configuration objects inherit from <a href="/docs/transformers/pr_30621/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/pr_30621/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> for more information.`,Se,j,we,U,rt="## JetMoeModel[[transformers.JetMoeModel]]",$e,T,A,Ie,se,st=`The bare JetMoe Model outputting raw hidden-states without any specific head on top.
This model is a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`,Oe,ae,at="Transformer decoder consisting of <em>config.num_hidden_layers</em> layers. Each layer is a <code>JetMoeBlock</code>",We,J,Z,He,ie,it='The <a href="/docs/transformers/pr_30621/en/model_doc/jetmoe#transformers.JetMoeModel">JetMoeModel</a> forward method, overrides the <code>__call__</code> special method.',Ne,S,ye,G,dt="## JetMoeForCausalLM[[transformers.JetMoeForCausalLM]]",xe,z,R,Ee,C,V,De,de,lt='The <a href="/docs/transformers/pr_30621/en/model_doc/jetmoe#transformers.JetMoeForCausalLM">JetMoeForCausalLM</a> forward method, overrides the <code>__call__</code> special method.',Be,I,ke,Y,ct="## JetMoeForSequenceClassification[[transformers.JetMoeForSequenceClassification]]",Je,u,Q,Ue,le,mt="The JetMoe Model transformer with a sequence classification head on top (linear layer).",Ae,ce,pt=`<a href="/docs/transformers/pr_30621/en/model_doc/jetmoe#transformers.JetMoeForSequenceClassification">JetMoeForSequenceClassification</a> uses the last token in order to do the classification, as other causal models
(e.g. GPT-2) do.`,Ze,me,ut=`Since it does classification on the last token, it requires to know the position of the last token. If a
<code>pad_token_id</code> is defined in the configuration, it finds the last token that is not a padding token in each row. If
no <code>pad_token_id</code> is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
padding tokens when <code>inputs_embeds</code> are passed instead of <code>input_ids</code>, it does the same (take the last value in
each row of the batch).`,Ge,pe,ft=`This model is a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`,Re,L,X,Ve,ue,ht='The <a href="/docs/transformers/pr_30621/en/model_doc/jetmoe#transformers.JetMoeForSequenceClassification">JetMoeForSequenceClassification</a> forward method, overrides the <code>__call__</code> special method.',Ye,O,Ce,K,Le,ge,ze;return W=new Xe({props:{title:"JetMoe",local:"jetmoe",headingTag:"h1"}}),H=new Xe({props:{title:"Overview",local:"overview",headingTag:"h2"}}),D=new Xe({props:{title:"JetMoeConfig",local:"transformers.JetMoeConfig",headingTag:"h2"}}),B=new te({props:{name:"class transformers.JetMoeConfig",anchor:"transformers.JetMoeConfig",parameters:[{name:"vocab_size",val:" = 32000"},{name:"hidden_size",val:" = 2048"},{name:"num_hidden_layers",val:" = 12"},{name:"num_key_value_heads",val:" = 16"},{name:"kv_channels",val:" = 128"},{name:"intermediate_size",val:" = 5632"},{name:"max_position_embeddings",val:" = 4096"},{name:"activation_function",val:" = 'silu'"},{name:"num_local_experts",val:" = 8"},{name:"num_experts_per_tok",val:" = 2"},{name:"output_router_logits",val:" = False"},{name:"aux_loss_coef",val:" = 0.01"},{name:"use_cache",val:" = True"},{name:"bos_token_id",val:" = 1"},{name:"eos_token_id",val:" = 2"},{name:"tie_word_embeddings",val:" = True"},{name:"rope_theta",val:" = 10000.0"},{name:"rms_norm_eps",val:" = 1e-06"},{name:"initializer_range",val:" = 0.01"},{name:"attention_dropout",val:" = 0.0"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.JetMoeConfig.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to 32000) &#x2014;
Vocabulary size of the JetMoe model. Defines the number of different tokens that can be represented by the
<code>inputs_ids</code> passed when calling <a href="/docs/transformers/pr_30621/en/model_doc/jetmoe#transformers.JetMoeModel">JetMoeModel</a>`,name:"vocab_size"},{anchor:"transformers.JetMoeConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 2048) &#x2014;
Dimension of the hidden representations.`,name:"hidden_size"},{anchor:"transformers.JetMoeConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 12) &#x2014;
Number of hidden layers in the Transformer encoder.`,name:"num_hidden_layers"},{anchor:"transformers.JetMoeConfig.num_key_value_heads",description:`<strong>num_key_value_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 16) &#x2014;
Number of attention heads for each key and value in the Transformer encoder.`,name:"num_key_value_heads"},{anchor:"transformers.JetMoeConfig.kv_channels",description:`<strong>kv_channels</strong> (<code>int</code>, <em>optional</em>, defaults to 128) &#x2014;
Defines the number of channels for the key and value tensors.`,name:"kv_channels"},{anchor:"transformers.JetMoeConfig.intermediate_size",description:`<strong>intermediate_size</strong> (<code>int</code>, <em>optional</em>, defaults to 5632) &#x2014;
Dimension of the MLP representations.`,name:"intermediate_size"},{anchor:"transformers.JetMoeConfig.max_position_embeddings",description:`<strong>max_position_embeddings</strong> (<code>int</code>, <em>optional</em>, defaults to 4096) &#x2014;
The maximum sequence length that this model might ever be used with. JetMoe&#x2019;s attention allows sequence of
up to 4096 tokens.`,name:"max_position_embeddings"},{anchor:"transformers.JetMoeConfig.activation_function",description:`<strong>activation_function</strong> (<code>string</code>, <em>optional</em>, defaults to <code>&quot;silu&quot;</code>) &#x2014;
Defines the activation function for MLP experts.`,name:"activation_function"},{anchor:"transformers.JetMoeConfig.num_local_experts",description:`<strong>num_local_experts</strong> (<code>int</code>, <em>optional</em>, defaults to 8) &#x2014;
Defines the number of experts in the MoE and MoA.`,name:"num_local_experts"},{anchor:"transformers.JetMoeConfig.num_experts_per_tok",description:"<strong>num_experts_per_tok</strong> (`int, <em>optional</em>, defaults to 2) &#x2014;\nThe number of experts to route per-token and for MoE and MoA.",name:"num_experts_per_tok"},{anchor:"transformers.JetMoeConfig.output_router_logits",description:`<strong>output_router_logits</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the router logits should be returned by the model. Enabeling this will also
allow the model to output the auxiliary loss.`,name:"output_router_logits"},{anchor:"transformers.JetMoeConfig.aux_loss_coef",description:`<strong>aux_loss_coef</strong> (<code>float</code>, <em>optional</em>, defaults to 0.01) &#x2014;
The coefficient for the auxiliary loss.`,name:"aux_loss_coef"},{anchor:"transformers.JetMoeConfig.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if <code>config.is_decoder=True</code>.`,name:"use_cache"},{anchor:"transformers.JetMoeConfig.bos_token_id",description:`<strong>bos_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The id of the &#x201C;beginning-of-sequence&#x201D; token.`,name:"bos_token_id"},{anchor:"transformers.JetMoeConfig.eos_token_id",description:`<strong>eos_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to 2) &#x2014;
The id of the &#x201C;end-of-sequence&#x201D; token.`,name:"eos_token_id"},{anchor:"transformers.JetMoeConfig.tie_word_embeddings",description:`<strong>tie_word_embeddings</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether the model&#x2019;s input and output word embeddings should be tied.`,name:"tie_word_embeddings"},{anchor:"transformers.JetMoeConfig.rope_theta",description:`<strong>rope_theta</strong> (<code>float</code>, <em>optional</em>, defaults to 10000.0) &#x2014;
The base period of the RoPE embeddings.`,name:"rope_theta"},{anchor:"transformers.JetMoeConfig.rms_norm_eps",description:`<strong>rms_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-06) &#x2014;
The epsilon used by the rms normalization layers.`,name:"rms_norm_eps"},{anchor:"transformers.JetMoeConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to 0.01) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.JetMoeConfig.attention_dropout",description:`<strong>attention_dropout</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The dropout ratio for the attention probabilities.`,name:"attention_dropout"}],source:"https://github.com/huggingface/transformers/blob/vr_30621/src/transformers/models/jetmoe/configuration_jetmoe.py#L24"}}),j=new wt({props:{anchor:"transformers.JetMoeConfig.example",$$slots:{default:[yt]},$$scope:{ctx:k}}}),A=new te({props:{name:"class transformers.JetMoeModel",anchor:"transformers.JetMoeModel",parameters:[{name:"config",val:": JetMoeConfig"}],parametersDescription:[{anchor:"transformers.JetMoeModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_30621/en/model_doc/jetmoe#transformers.JetMoeConfig">JetMoeConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/pr_30621/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.
config &#x2014;
JetMoeConfig`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/vr_30621/src/transformers/models/jetmoe/modeling_jetmoe.py#L1011"}}),Z=new te({props:{name:"forward",anchor:"transformers.JetMoeModel.forward",parameters:[{name:"input_ids",val:": LongTensor = None"},{name:"attention_mask",val:": Optional = None"},{name:"position_ids",val:": Optional = None"},{name:"past_key_values",val:": Union = None"},{name:"inputs_embeds",val:": Optional = None"},{name:"use_cache",val:": Optional = None"},{name:"output_attentions",val:": Optional = None"},{name:"output_hidden_states",val:": Optional = None"},{name:"output_router_logits",val:": Optional = None"},{name:"return_dict",val:": Optional = None"},{name:"cache_position",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.JetMoeModel.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>({0})</code>) &#x2014;
Indices of input sequence tokens in the vocabulary.</p>
<p>Indices can be obtained using <code>AutoProcenizer</code>. See <a href="/docs/transformers/pr_30621/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/pr_30621/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.JetMoeModel.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.FloatTensor</code> of shape <code>({0})</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.JetMoeModel.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>({0})</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p>
<p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.JetMoeModel.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>({0}, hidden_dim)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <em>input_ids</em> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.JetMoeModel.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.JetMoeModel.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.JetMoeModel.forward.output_router_logits",description:`<strong>output_router_logits</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
should not be returned during inference.`,name:"output_router_logits"},{anchor:"transformers.JetMoeModel.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/pr_30621/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.JetMoeModel.forward.cache_position",description:`<strong>cache_position</strong> (<code>torch.LongTensor</code> of shape <code>(sequence_length)</code>, <em>optional</em>) &#x2014;
Indices depicting the position of the input sequence tokens in the sequence. Contrarily to <code>position_ids</code>,
this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
the complete sequence length.`,name:"cache_position"}],source:"https://github.com/huggingface/transformers/blob/vr_30621/src/transformers/models/jetmoe/modeling_jetmoe.py#L1046"}}),S=new Qe({props:{$$slots:{default:[xt]},$$scope:{ctx:k}}}),R=new te({props:{name:"class transformers.JetMoeForCausalLM",anchor:"transformers.JetMoeForCausalLM",parameters:[{name:"config",val:""}],source:"https://github.com/huggingface/transformers/blob/vr_30621/src/transformers/models/jetmoe/modeling_jetmoe.py#L1259"}}),V=new te({props:{name:"forward",anchor:"transformers.JetMoeForCausalLM.forward",parameters:[{name:"input_ids",val:": LongTensor = None"},{name:"attention_mask",val:": Optional = None"},{name:"position_ids",val:": Optional = None"},{name:"past_key_values",val:": Optional = None"},{name:"inputs_embeds",val:": Optional = None"},{name:"labels",val:": Optional = None"},{name:"use_cache",val:": Optional = None"},{name:"output_attentions",val:": Optional = None"},{name:"output_hidden_states",val:": Optional = None"},{name:"output_router_logits",val:": Optional = None"},{name:"return_dict",val:": Optional = None"},{name:"cache_position",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.JetMoeForCausalLM.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>({0})</code>) &#x2014;
Indices of input sequence tokens in the vocabulary.</p>
<p>Indices can be obtained using <code>AutoProcenizer</code>. See <a href="/docs/transformers/pr_30621/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/pr_30621/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.JetMoeForCausalLM.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.FloatTensor</code> of shape <code>({0})</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.JetMoeForCausalLM.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>({0})</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p>
<p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.JetMoeForCausalLM.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>({0}, hidden_dim)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <em>input_ids</em> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.JetMoeForCausalLM.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.JetMoeForCausalLM.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.JetMoeForCausalLM.forward.output_router_logits",description:`<strong>output_router_logits</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
should not be returned during inference.`,name:"output_router_logits"},{anchor:"transformers.JetMoeForCausalLM.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/pr_30621/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.JetMoeForCausalLM.forward.cache_position",description:`<strong>cache_position</strong> (<code>torch.LongTensor</code> of shape <code>(sequence_length)</code>, <em>optional</em>) &#x2014;
Indices depicting the position of the input sequence tokens in the sequence. Contrarily to <code>position_ids</code>,
this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
the complete sequence length.</p>
<p>Args &#x2014;
labels (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>):
Labels for computing the masked language modeling loss. Indices should either be in <code>[0, ..., config.vocab_size]</code> or -100 (see <code>input_ids</code> docstring). Tokens with indices set to <code>-100</code> are ignored
(masked), the loss is only computed for the tokens with labels in <code>[0, ..., config.vocab_size]</code>.`,name:"cache_position"}],source:"https://github.com/huggingface/transformers/blob/vr_30621/src/transformers/models/jetmoe/modeling_jetmoe.py#L1297",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>transformers.modeling_outputs.MoeCausalLMOutputWithPast</code> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/pr_30621/en/model_doc/jetmoe#transformers.JetMoeConfig"
>JetMoeConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Language modeling loss (for next-token prediction).</p>
</li>
<li>
<p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) — Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p>
</li>
<li>
<p><strong>aux_loss</strong> (<code>torch.FloatTensor</code>, <em>optional</em>, returned when <code>labels</code> is provided) — aux_loss for the sparse modules.</p>
</li>
<li>
<p><strong>router_logits</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_router_probs=True</code> and <code>config.add_router_probs=True</code> is passed or when <code>config.output_router_probs=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, sequence_length, num_experts)</code>.</p>
<p>Raw router logtis (post-softmax) that are computed by MoE routers, these terms are used to compute the auxiliary
loss for Mixture of Experts models.</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) — Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>)</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
<code>past_key_values</code> input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p>
</li>
<li>
<p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p>
</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>transformers.modeling_outputs.MoeCausalLMOutputWithPast</code> or <code>tuple(torch.FloatTensor)</code></p>
`}}),I=new Qe({props:{$$slots:{default:[kt]},$$scope:{ctx:k}}}),Q=new te({props:{name:"class transformers.JetMoeForSequenceClassification",anchor:"transformers.JetMoeForSequenceClassification",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.JetMoeForSequenceClassification.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_30621/en/model_doc/jetmoe#transformers.JetMoeConfig">JetMoeConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/pr_30621/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/vr_30621/src/transformers/models/jetmoe/modeling_jetmoe.py#L1485"}}),X=new te({props:{name:"forward",anchor:"transformers.JetMoeForSequenceClassification.forward",parameters:[{name:"input_ids",val:": LongTensor = None"},{name:"attention_mask",val:": Optional = None"},{name:"position_ids",val:": Optional = None"},{name:"past_key_values",val:": Union = None"},{name:"inputs_embeds",val:": Optional = None"},{name:"labels",val:": Optional = None"},{name:"use_cache",val:": Optional = None"},{name:"output_attentions",val:": Optional = None"},{name:"output_hidden_states",val:": Optional = None"},{name:"return_dict",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.JetMoeForSequenceClassification.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>({0})</code>) &#x2014;
Indices of input sequence tokens in the vocabulary.</p>
<p>Indices can be obtained using <code>AutoProcenizer</code>. See <a href="/docs/transformers/pr_30621/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/pr_30621/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.JetMoeForSequenceClassification.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.FloatTensor</code> of shape <code>({0})</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.JetMoeForSequenceClassification.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>({0})</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p>
<p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.JetMoeForSequenceClassification.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>({0}, hidden_dim)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <em>input_ids</em> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.JetMoeForSequenceClassification.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.JetMoeForSequenceClassification.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.JetMoeForSequenceClassification.forward.output_router_logits",description:`<strong>output_router_logits</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
should not be returned during inference.`,name:"output_router_logits"},{anchor:"transformers.JetMoeForSequenceClassification.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/pr_30621/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.JetMoeForSequenceClassification.forward.cache_position",description:`<strong>cache_position</strong> (<code>torch.LongTensor</code> of shape <code>(sequence_length)</code>, <em>optional</em>) &#x2014;
Indices depicting the position of the input sequence tokens in the sequence. Contrarily to <code>position_ids</code>,
this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
the complete sequence length.`,name:"cache_position"},{anchor:"transformers.JetMoeForSequenceClassification.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size,)</code>, <em>optional</em>) &#x2014;
Labels for computing the sequence classification/regression loss. Indices should be in <code>[0, ..., config.num_labels - 1]</code>. If <code>config.num_labels == 1</code> a regression loss is computed (Mean-Square loss), If
<code>config.num_labels &gt; 1</code> a classification loss is computed (Cross-Entropy).`,name:"labels"}],source:"https://github.com/huggingface/transformers/blob/vr_30621/src/transformers/models/jetmoe/modeling_jetmoe.py#L1517"}}),O=new Qe({props:{$$slots:{default:[Jt]},$$scope:{ctx:k}}}),K=new $t({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/jetmoe.md"}}),{c(){o=d("meta"),p=s(),i=d("p"),$=s(),f(W.$$.fragment),_e=s(),f(H.$$.fragment),Me=s(),N=d("p"),N.innerHTML=Ke,be=s(),E=d("p"),E.innerHTML=et,ve=s(),f(D.$$.fragment),Te=s(),v=d("div"),f(B.$$.fragment),Pe=s(),oe=d("p"),oe.innerHTML=tt,qe=s(),ne=d("p"),ne.innerHTML=ot,je=s(),re=d("p"),re.innerHTML=nt,Se=s(),f(j.$$.fragment),we=s(),U=d("p"),U.textContent=rt,$e=s(),T=d("div"),f(A.$$.fragment),Ie=s(),se=d("p"),se.innerHTML=st,Oe=s(),ae=d("p"),ae.innerHTML=at,We=s(),J=d("div"),f(Z.$$.fragment),He=s(),ie=d("p"),ie.innerHTML=it,Ne=s(),f(S.$$.fragment),ye=s(),G=d("p"),G.textContent=dt,xe=s(),z=d("div"),f(R.$$.fragment),Ee=s(),C=d("div"),f(V.$$.fragment),De=s(),de=d("p"),de.innerHTML=lt,Be=s(),f(I.$$.fragment),ke=s(),Y=d("p"),Y.textContent=ct,Je=s(),u=d("div"),f(Q.$$.fragment),Ue=s(),le=d("p"),le.textContent=mt,Ae=s(),ce=d("p"),ce.innerHTML=pt,Ze=s(),me=d("p"),me.innerHTML=ut,Ge=s(),pe=d("p"),pe.innerHTML=ft,Re=s(),L=d("div"),f(X.$$.fragment),Ve=s(),ue=d("p"),ue.innerHTML=ht,Ye=s(),f(O.$$.fragment),Ce=s(),f(K.$$.fragment),Le=s(),ge=d("p"),this.h()},l(e){const t=vt("svelte-u9bgzb",document.head);o=l(t,"META",{name:!0,content:!0}),t.forEach(n),p=a(e),i=l(e,"P",{}),P(i).forEach(n),$=a(e),h(W.$$.fragment,e),_e=a(e),h(H.$$.fragment,e),Me=a(e),N=l(e,"P",{"data-svelte-h":!0}),m(N)!=="svelte-o95nje"&&(N.innerHTML=Ke),be=a(e),E=l(e,"P",{"data-svelte-h":!0}),m(E)!=="svelte-8foy02"&&(E.innerHTML=et),ve=a(e),h(D.$$.fragment,e),Te=a(e),v=l(e,"DIV",{class:!0});var w=P(v);h(B.$$.fragment,w),Pe=a(w),oe=l(w,"P",{"data-svelte-h":!0}),m(oe)!=="svelte-1g120fb"&&(oe.innerHTML=tt),qe=a(w),ne=l(w,"P",{"data-svelte-h":!0}),m(ne)!=="svelte-aqfqqx"&&(ne.innerHTML=ot),je=a(w),re=l(w,"P",{"data-svelte-h":!0}),m(re)!=="svelte-pf84nr"&&(re.innerHTML=nt),Se=a(w),h(j.$$.fragment,w),w.forEach(n),we=a(e),U=l(e,"P",{"data-svelte-h":!0}),m(U)!=="svelte-1de2kr4"&&(U.textContent=rt),$e=a(e),T=l(e,"DIV",{class:!0});var x=P(T);h(A.$$.fragment,x),Ie=a(x),se=l(x,"P",{"data-svelte-h":!0}),m(se)!=="svelte-rp8i7n"&&(se.innerHTML=st),Oe=a(x),ae=l(x,"P",{"data-svelte-h":!0}),m(ae)!=="svelte-1yw4f7n"&&(ae.innerHTML=at),We=a(x),J=l(x,"DIV",{class:!0});var F=P(J);h(Z.$$.fragment,F),He=a(F),ie=l(F,"P",{"data-svelte-h":!0}),m(ie)!=="svelte-2rmit5"&&(ie.innerHTML=it),Ne=a(F),h(S.$$.fragment,F),F.forEach(n),x.forEach(n),ye=a(e),G=l(e,"P",{"data-svelte-h":!0}),m(G)!=="svelte-1hcfdo4"&&(G.textContent=dt),xe=a(e),z=l(e,"DIV",{class:!0});var ee=P(z);h(R.$$.fragment,ee),Ee=a(ee),C=l(ee,"DIV",{class:!0});var fe=P(C);h(V.$$.fragment,fe),De=a(fe),de=l(fe,"P",{"data-svelte-h":!0}),m(de)!=="svelte-1jh8u7x"&&(de.innerHTML=lt),Be=a(fe),h(I.$$.fragment,fe),fe.forEach(n),ee.forEach(n),ke=a(e),Y=l(e,"P",{"data-svelte-h":!0}),m(Y)!=="svelte-d9ndo8"&&(Y.textContent=ct),Je=a(e),u=l(e,"DIV",{class:!0});var y=P(u);h(Q.$$.fragment,y),Ue=a(y),le=l(y,"P",{"data-svelte-h":!0}),m(le)!=="svelte-1pmdz94"&&(le.textContent=mt),Ae=a(y),ce=l(y,"P",{"data-svelte-h":!0}),m(ce)!=="svelte-adjzis"&&(ce.innerHTML=pt),Ze=a(y),me=l(y,"P",{"data-svelte-h":!0}),m(me)!=="svelte-10ugs3m"&&(me.innerHTML=ut),Ge=a(y),pe=l(y,"P",{"data-svelte-h":!0}),m(pe)!=="svelte-1ytzpa1"&&(pe.innerHTML=ft),Re=a(y),L=l(y,"DIV",{class:!0});var he=P(L);h(X.$$.fragment,he),Ve=a(he),ue=l(he,"P",{"data-svelte-h":!0}),m(ue)!=="svelte-11jaka1"&&(ue.innerHTML=ht),Ye=a(he),h(O.$$.fragment,he),he.forEach(n),y.forEach(n),Ce=a(e),h(K.$$.fragment,e),Le=a(e),ge=l(e,"P",{}),P(ge).forEach(n),this.h()},h(){q(o,"name","hf:doc:metadata"),q(o,"content",Lt),q(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),q(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),q(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),q(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),q(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),q(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),q(u,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){r(document.head,o),c(e,p,t),c(e,i,t),c(e,$,t),g(W,e,t),c(e,_e,t),g(H,e,t),c(e,Me,t),c(e,N,t),c(e,be,t),c(e,E,t),c(e,ve,t),g(D,e,t),c(e,Te,t),c(e,v,t),g(B,v,null),r(v,Pe),r(v,oe),r(v,qe),r(v,ne),r(v,je),r(v,re),r(v,Se),g(j,v,null),c(e,we,t),c(e,U,t),c(e,$e,t),c(e,T,t),g(A,T,null),r(T,Ie),r(T,se),r(T,Oe),r(T,ae),r(T,We),r(T,J),g(Z,J,null),r(J,He),r(J,ie),r(J,Ne),g(S,J,null),c(e,ye,t),c(e,G,t),c(e,xe,t),c(e,z,t),g(R,z,null),r(z,Ee),r(z,C),g(V,C,null),r(C,De),r(C,de),r(C,Be),g(I,C,null),c(e,ke,t),c(e,Y,t),c(e,Je,t),c(e,u,t),g(Q,u,null),r(u,Ue),r(u,le),r(u,Ae),r(u,ce),r(u,Ze),r(u,me),r(u,Ge),r(u,pe),r(u,Re),r(u,L),g(X,L,null),r(L,Ve),r(L,ue),r(L,Ye),g(O,L,null),c(e,Ce,t),g(K,e,t),c(e,Le,t),c(e,ge,t),ze=!0},p(e,[t]){const w={};t&2&&(w.$$scope={dirty:t,ctx:e}),j.$set(w);const x={};t&2&&(x.$$scope={dirty:t,ctx:e}),S.$set(x);const F={};t&2&&(F.$$scope={dirty:t,ctx:e}),I.$set(F);const ee={};t&2&&(ee.$$scope={dirty:t,ctx:e}),O.$set(ee)},i(e){ze||(_(W.$$.fragment,e),_(H.$$.fragment,e),_(D.$$.fragment,e),_(B.$$.fragment,e),_(j.$$.fragment,e),_(A.$$.fragment,e),_(Z.$$.fragment,e),_(S.$$.fragment,e),_(R.$$.fragment,e),_(V.$$.fragment,e),_(I.$$.fragment,e),_(Q.$$.fragment,e),_(X.$$.fragment,e),_(O.$$.fragment,e),_(K.$$.fragment,e),ze=!0)},o(e){M(W.$$.fragment,e),M(H.$$.fragment,e),M(D.$$.fragment,e),M(B.$$.fragment,e),M(j.$$.fragment,e),M(A.$$.fragment,e),M(Z.$$.fragment,e),M(S.$$.fragment,e),M(R.$$.fragment,e),M(V.$$.fragment,e),M(I.$$.fragment,e),M(Q.$$.fragment,e),M(X.$$.fragment,e),M(O.$$.fragment,e),M(K.$$.fragment,e),ze=!1},d(e){e&&(n(p),n(i),n($),n(_e),n(Me),n(N),n(be),n(E),n(ve),n(Te),n(v),n(we),n(U),n($e),n(T),n(ye),n(G),n(xe),n(z),n(ke),n(Y),n(Je),n(u),n(Ce),n(Le),n(ge)),n(o),b(W,e),b(H,e),b(D,e),b(B),b(j),b(A),b(Z),b(S),b(R),b(V),b(I),b(Q),b(X),b(O),b(K,e)}}}const Lt='{"title":"JetMoe","local":"jetmoe","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"JetMoeConfig","local":"transformers.JetMoeConfig","sections":[],"depth":2}],"depth":1}';function zt(k){return _t(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Wt extends Mt{constructor(o){super(),bt(this,o,zt,Ct,gt,{})}}export{Wt as component};
