import{s as Ke,o as et,n as ze}from"../chunks/scheduler.25b97de1.js";import{S as tt,i as nt,g as l,s as i,r as _,A as at,h as d,f as n,c as r,j as ae,u as v,x as h,k as oe,y as m,a,v as b,d as M,t as y,w as T}from"../chunks/index.d9030fc9.js";import{T as ot}from"../chunks/Tip.baa67368.js";import{D as Fe}from"../chunks/Docstring.e257edda.js";import{C as je}from"../chunks/CodeBlock.e6cd0d95.js";import{E as Oe}from"../chunks/ExampleCodeBlock.20db4b6e.js";import{H as we,E as st}from"../chunks/EditOnGithub.91d95064.js";function it(k){let s,w="Example:",p,c,u;return c=new je({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFZpcExsYXZhRm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTJDJTIwVmlwTGxhdmFDb25maWclMkMlMjBDTElQVmlzaW9uQ29uZmlnJTJDJTIwTGxhbWFDb25maWclMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwQ0xJUC12aXNpb24lMjBjb25maWclMEF2aXNpb25fY29uZmlnJTIwJTNEJTIwQ0xJUFZpc2lvbkNvbmZpZygpJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMExsYW1hJTIwY29uZmlnJTBBdGV4dF9jb25maWclMjAlM0QlMjBMbGFtYUNvbmZpZygpJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMFZpcExsYXZhJTIwdmlwbGxhdmEtN2IlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFjb25maWd1cmF0aW9uJTIwJTNEJTIwVmlwTGxhdmFDb25maWcodmlzaW9uX2NvbmZpZyUyQyUyMHRleHRfY29uZmlnKSUwQSUwQSUyMyUyMEluaXRpYWxpemluZyUyMGElMjBtb2RlbCUyMGZyb20lMjB0aGUlMjB2aXBsbGF2YS03YiUyMHN0eWxlJTIwY29uZmlndXJhdGlvbiUwQW1vZGVsJTIwJTNEJTIwVmlwTGxhdmFGb3JDb25kaXRpb25hbEdlbmVyYXRpb24oY29uZmlndXJhdGlvbiklMEElMEElMjMlMjBBY2Nlc3NpbmclMjB0aGUlMjBtb2RlbCUyMGNvbmZpZ3VyYXRpb24lMEFjb25maWd1cmF0aW9uJTIwJTNEJTIwbW9kZWwuY29uZmln",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> VipLlavaForConditionalGeneration, VipLlavaConfig, CLIPVisionConfig, LlamaConfig

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a CLIP-vision config</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>vision_config = CLIPVisionConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a Llama config</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>text_config = LlamaConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a VipLlava vipllava-7b style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = VipLlavaConfig(vision_config, text_config)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a model from the vipllava-7b style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = VipLlavaForConditionalGeneration(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`,wrap:!1}}),{c(){s=l("p"),s.textContent=w,p=i(),_(c.$$.fragment)},l(o){s=d(o,"P",{"data-svelte-h":!0}),h(s)!=="svelte-11lpom8"&&(s.textContent=w),p=r(o),v(c.$$.fragment,o)},m(o,g){a(o,s,g),a(o,p,g),b(c,o,g),u=!0},p:ze,i(o){u||(M(c.$$.fragment,o),u=!0)},o(o){y(c.$$.fragment,o),u=!1},d(o){o&&(n(s),n(p)),T(c,o)}}}function rt(k){let s,w=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){s=l("p"),s.innerHTML=w},l(p){s=d(p,"P",{"data-svelte-h":!0}),h(s)!=="svelte-fincs2"&&(s.innerHTML=w)},m(p,c){a(p,s,c)},p:ze,d(p){p&&n(s)}}}function lt(k){let s,w="Example:",p,c,u;return c=new je({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwUElMJTIwaW1wb3J0JTIwSW1hZ2UlMEFpbXBvcnQlMjByZXF1ZXN0cyUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvUHJvY2Vzc29yJTJDJTIwVmlwTGxhdmFGb3JDb25kaXRpb25hbEdlbmVyYXRpb24lMEElMEFtb2RlbCUyMCUzRCUyMFZpcExsYXZhRm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uLmZyb21fcHJldHJhaW5lZCglMjJsbGF2YS1oZiUyRnZpcC1sbGF2YS03Yi1oZiUyMiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2KSUwQXByb2Nlc3NvciUyMCUzRCUyMEF1dG9Qcm9jZXNzb3IuZnJvbV9wcmV0cmFpbmVkKCUyMmxsYXZhLWhmJTJGdmlwLWxsYXZhLTdiLWhmJTIyKSUwQSUwQXByb21wdCUyMCUzRCUyMCUyMkElMjBjaGF0JTIwYmV0d2VlbiUyMGElMjBjdXJpb3VzJTIwaHVtYW4lMjBhbmQlMjBhbiUyMGFydGlmaWNpYWwlMjBpbnRlbGxpZ2VuY2UlMjBhc3Npc3RhbnQuJTIwVGhlJTIwYXNzaXN0YW50JTIwZ2l2ZXMlMjBoZWxwZnVsJTJDJTIwZGV0YWlsZWQlMkMlMjBhbmQlMjBwb2xpdGUlMjBhbnN3ZXJzJTIwdG8lMjB0aGUlMjBodW1hbidzJTIwcXVlc3Rpb25zLiUyMyUyMyUyM0h1bWFuJTNBJTIwJTNDaW1hZ2UlM0UlNUNuJTdCJTdEJTIzJTIzJTIzQXNzaXN0YW50JTNBJTIyJTBBcXVlc3Rpb24lMjAlM0QlMjAlMjJDYW4lMjB5b3UlMjBwbGVhc2UlMjBkZXNjcmliZSUyMHRoaXMlMjBpbWFnZSUzRiUyMiUwQXByb21wdCUyMCUzRCUyMHByb21wdC5mb3JtYXQocXVlc3Rpb24pJTBBdXJsJTIwJTNEJTIwJTIyaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmRhdGFzZXRzJTJGaHVnZ2luZ2ZhY2UlMkZkb2N1bWVudGF0aW9uLWltYWdlcyUyRnJlc29sdmUlMkZtYWluJTJGZGlmZnVzZXJzJTJGY29tcGVsLW5lZy5wbmclMjIlMEFpbWFnZSUyMCUzRCUyMEltYWdlLm9wZW4ocmVxdWVzdHMuZ2V0KHVybCUyQyUyMHN0cmVhbSUzRFRydWUpLnJhdyklMEElMEFpbnB1dHMlMjAlM0QlMjBwcm9jZXNzb3IodGV4dCUzRHRleHQlMkMlMjBpbWFnZXMlM0RpbWFnZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKDAlMkMlMjB0b3JjaC5mbG9hdDE2KSUwQSUwQSUyMyUyMEdlbmVyYXRlJTBBZ2VuZXJhdGVfaWRzJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDIwKSUwQXByb2Nlc3Nvci5kZWNvZGUoZ2VuZXJhdGVfaWRzJTVCMCU1RCU1QmxlbihpbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQlNUIwJTVEKSUzQSU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> requests
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, VipLlavaForConditionalGeneration

<span class="hljs-meta">&gt;&gt;&gt; </span>model = VipLlavaForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;llava-hf/vip-llava-7b-hf&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, torch_dtype=torch.float16)
<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;llava-hf/vip-llava-7b-hf&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human&#x27;s questions.###Human: &lt;image&gt;\\n{}###Assistant:&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>question = <span class="hljs-string">&quot;Can you please describe this image?&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = prompt.<span class="hljs-built_in">format</span>(question)
<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/compel-neg.png&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = processor(text=text, images=image, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-number">0</span>, torch.float16)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Generate</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generate_ids = model.generate(**inputs, max_new_tokens=<span class="hljs-number">20</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>processor.decode(generate_ids[<span class="hljs-number">0</span>][<span class="hljs-built_in">len</span>(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>][<span class="hljs-number">0</span>]):], skip_special_tokens=<span class="hljs-literal">True</span>)
The image features a brown <span class="hljs-keyword">and</span> white cat sitting on a green surface, <span class="hljs-keyword">with</span> a red ball <span class="hljs-keyword">in</span> its`,wrap:!1}}),{c(){s=l("p"),s.textContent=w,p=i(),_(c.$$.fragment)},l(o){s=d(o,"P",{"data-svelte-h":!0}),h(s)!=="svelte-11lpom8"&&(s.textContent=w),p=r(o),v(c.$$.fragment,o)},m(o,g){a(o,s,g),a(o,p,g),b(c,o,g),u=!0},p:ze,i(o){u||(M(c.$$.fragment,o),u=!0)},o(o){y(c.$$.fragment,o),u=!1},d(o){o&&(n(s),n(p)),T(c,o)}}}function dt(k){let s,w,p,c,u,o,g,se,G,We='The VipLlava model was proposed in <a href="https://arxiv.org/abs/2312.00784" rel="nofollow">Making Large Multimodal Models Understand Arbitrary Visual Prompts</a> by Mu Cai, Haotian Liu, Siva Karthik Mustikovela, Gregory P. Meyer, Yuning Chai, Dennis Park, Yong Jae Lee.',ie,Z,Ie="VipLlava enhances the training protocol of Llava by marking images and interact with the model using natural cues like a “red bounding box” or “pointed arrow” during training.",re,F,Be="The abstract from the paper is the following:",le,z,Ne="<em>While existing large vision-language multimodal models focus on whole image understanding, there is a prominent gap in achieving region-specific comprehension. Current approaches that use textual coordinates or spatial encodings often fail to provide a user-friendly interface for visual prompting. To address this challenge, we introduce a novel multimodal model capable of decoding arbitrary visual prompts. This allows users to intuitively mark images and interact with the model using natural cues like a “red bounding box” or “pointed arrow”. Our simple design directly overlays visual markers onto the RGB image, eliminating the need for complex region encodings, yet achieves state-of-the-art performance on region-understanding tasks like Visual7W, PointQA, and Visual Commonsense Reasoning benchmark. Furthermore, we present ViP-Bench, a comprehensive benchmark to assess the capability of models in understanding visual prompts across multiple dimensions, enabling future research in this domain. Code, data, and model are publicly available.</em>",de,W,He="Tips:",ce,I,Xe="<li><p>The architecture is similar than llava architecture except that the multi-modal projector takes a set of concatenated vision hidden states and has an additional layernorm layer on that module.</p></li> <li><p>We advise users to use <code>padding_side=&quot;left&quot;</code> when computing batched generation as it leads to more accurate results. Simply make sure to call <code>processor.tokenizer.padding_side = &quot;left&quot;</code> before generating.</p></li> <li><p>Note the model has not been explicitly trained to process multiple images in the same prompt, although this is technically possible, you may experience inaccurate results.</p></li> <li><p>For better results, we recommend users to prompt the model with the correct prompt format:</p></li>",pe,B,me,N,Pe="For multiple turns conversation:",he,H,ue,X,Re='The original code can be found <a href="https://github.com/mu-cai/ViP-LLaVA" rel="nofollow">here</a>.',ge,P,Ye='This model was contributed by <a href="https://huggingface.co/ybelkada" rel="nofollow">Younes Belkada</a>',fe,R,_e,f,Y,Ce,A,Ee=`This is the configuration class to store the configuration of a <a href="/docs/transformers/pr_30929/en/model_doc/vipllava#transformers.VipLlavaForConditionalGeneration">VipLlavaForConditionalGeneration</a>. It is used to instantiate an
VipLlava model according to the specified arguments, defining the model architecture. Instantiating a configuration
with the defaults will yield a similar configuration to that of the VipLlava-9B.`,Ue,D,Se='e.g. <a href="https://huggingface.co/ybelkada/vip-llava-7b-hf" rel="nofollow">ybelkada/vip-llava-7b-hf</a>',Je,O,Qe=`Configuration objects inherit from <a href="/docs/transformers/pr_30929/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/pr_30929/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> for more information.`,$e,x,ve,E,be,j,S,ke,K,qe=`The VIPLLAVA model which consists of a vision backbone and a language model.
This model inherits from <a href="/docs/transformers/pr_30929/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`,xe,ee,Ae=`This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`,Ve,U,Q,Le,te,De='The <a href="/docs/transformers/pr_30929/en/model_doc/vipllava#transformers.VipLlavaForConditionalGeneration">VipLlavaForConditionalGeneration</a> forward method, overrides the <code>__call__</code> special method.',Ge,V,Ze,L,Me,q,ye,ne,Te;return u=new we({props:{title:"VipLlava",local:"vipllava",headingTag:"h1"}}),g=new we({props:{title:"Overview",local:"overview",headingTag:"h2"}}),B=new je({props:{code:"QSUyMGNoYXQlMjBiZXR3ZWVuJTIwYSUyMGN1cmlvdXMlMjBodW1hbiUyMGFuZCUyMGFuJTIwYXJ0aWZpY2lhbCUyMGludGVsbGlnZW5jZSUyMGFzc2lzdGFudC4lMjBUaGUlMjBhc3Npc3RhbnQlMjBnaXZlcyUyMGhlbHBmdWwlMkMlMjBkZXRhaWxlZCUyQyUyMGFuZCUyMHBvbGl0ZSUyMGFuc3dlcnMlMjB0byUyMHRoZSUyMGh1bWFuJ3MlMjBxdWVzdGlvbnMuJTIzJTIzJTIzSHVtYW4lM0ElMjAlM0NpbWFnZSUzRSU1Q24lM0Nwcm9tcHQlM0UlMjMlMjMlMjNBc3Npc3RhbnQlM0E=",highlighted:'A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human<span class="hljs-string">&#x27;s questions.###Human: &lt;image&gt;\\n&lt;prompt&gt;###Assistant:</span>',wrap:!1}}),H=new je({props:{code:"QSUyMGNoYXQlMjBiZXR3ZWVuJTIwYSUyMGN1cmlvdXMlMjBodW1hbiUyMGFuZCUyMGFuJTIwYXJ0aWZpY2lhbCUyMGludGVsbGlnZW5jZSUyMGFzc2lzdGFudC4lMjBUaGUlMjBhc3Npc3RhbnQlMjBnaXZlcyUyMGhlbHBmdWwlMkMlMjBkZXRhaWxlZCUyQyUyMGFuZCUyMHBvbGl0ZSUyMGFuc3dlcnMlMjB0byUyMHRoZSUyMGh1bWFuJ3MlMjBxdWVzdGlvbnMuJTIzJTIzJTIzSHVtYW4lM0ElMjAlM0NpbWFnZSUzRSU1Q24lM0Nwcm9tcHQxJTNFJTIzJTIzJTIzQXNzaXN0YW50JTNBJTIwJTNDYW5zd2VyMSUzRSUyMyUyMyUyM0h1bWFuJTNBJTIwJTNDcHJvbXB0MiUzRSUyMyUyMyUyM0Fzc2lzdGFudCUzQQ==",highlighted:'A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human<span class="hljs-string">&#x27;s questions.###Human: &lt;image&gt;\\n&lt;prompt1&gt;###Assistant: &lt;answer1&gt;###Human: &lt;prompt2&gt;###Assistant:</span>',wrap:!1}}),R=new we({props:{title:"VipLlavaConfig",local:"transformers.VipLlavaConfig",headingTag:"h2"}}),Y=new Fe({props:{name:"class transformers.VipLlavaConfig",anchor:"transformers.VipLlavaConfig",parameters:[{name:"vision_config",val:" = None"},{name:"text_config",val:" = None"},{name:"ignore_index",val:" = -100"},{name:"image_token_index",val:" = 32000"},{name:"projector_hidden_act",val:" = 'gelu'"},{name:"projector_layernorm_eps",val:" = 1e-05"},{name:"vision_feature_layers",val:" = [-2, -5, -8, -11, 6]"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.VipLlavaConfig.vision_config",description:`<strong>vision_config</strong> (<code>VipLlavaVisionConfig</code>,  <em>optional</em>) &#x2014;
Custom vision config or dict`,name:"vision_config"},{anchor:"transformers.VipLlavaConfig.text_config",description:`<strong>text_config</strong> (<code>Union[AutoConfig, dict]</code>, <em>optional</em>) &#x2014;
The config object of the text backbone. Can be any of <code>LlamaConfig</code> or <code>MistralConfig</code>.`,name:"text_config"},{anchor:"transformers.VipLlavaConfig.ignore_index",description:`<strong>ignore_index</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The ignore index for the loss function.`,name:"ignore_index"},{anchor:"transformers.VipLlavaConfig.image_token_index",description:`<strong>image_token_index</strong> (<code>int</code>, <em>optional</em>, defaults to 32000) &#x2014;
The image token index to encode the image prompt.`,name:"image_token_index"},{anchor:"transformers.VipLlavaConfig.projector_hidden_act",description:`<strong>projector_hidden_act</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;gelu&quot;</code>) &#x2014;
The activation function used by the multimodal projector.`,name:"projector_hidden_act"},{anchor:"transformers.VipLlavaConfig.projector_layernorm_eps",description:`<strong>projector_layernorm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-05) &#x2014;
The layer norm epsilon of the projector layernorm`,name:"projector_layernorm_eps"},{anchor:"transformers.VipLlavaConfig.vision_feature_layers",description:`<strong>vision_feature_layers</strong> (<code>List[int]</code>, <em>optional</em>, defaults to <code>[-2, -5, -8, -11, 6]</code>) &#x2014;
The list of layers to select the vision features from.`,name:"vision_feature_layers"}],source:"https://github.com/huggingface/transformers/blob/vr_30929/src/transformers/models/vipllava/configuration_vipllava.py#L26"}}),x=new Oe({props:{anchor:"transformers.VipLlavaConfig.example",$$slots:{default:[it]},$$scope:{ctx:k}}}),E=new we({props:{title:"VipLlavaForConditionalGeneration",local:"transformers.VipLlavaForConditionalGeneration",headingTag:"h2"}}),S=new Fe({props:{name:"class transformers.VipLlavaForConditionalGeneration",anchor:"transformers.VipLlavaForConditionalGeneration",parameters:[{name:"config",val:": VipLlavaConfig"}],parametersDescription:[{anchor:"transformers.VipLlavaForConditionalGeneration.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_30929/en/model_doc/vipllava#transformers.VipLlavaConfig">VipLlavaConfig</a> or <code>VipLlavaVisionConfig</code>) &#x2014;
Model configuration class with all the parameters of the model. Initializing with a config file does not
load the weights associated with the model, only the configuration. Check out the
<a href="/docs/transformers/pr_30929/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/vr_30929/src/transformers/models/vipllava/modeling_vipllava.py#L236"}}),Q=new Fe({props:{name:"forward",anchor:"transformers.VipLlavaForConditionalGeneration.forward",parameters:[{name:"input_ids",val:": LongTensor = None"},{name:"pixel_values",val:": FloatTensor = None"},{name:"attention_mask",val:": Optional = None"},{name:"position_ids",val:": Optional = None"},{name:"past_key_values",val:": Optional = None"},{name:"inputs_embeds",val:": Optional = None"},{name:"vision_feature_layers",val:": Optional = None"},{name:"labels",val:": Optional = None"},{name:"use_cache",val:": Optional = None"},{name:"output_attentions",val:": Optional = None"},{name:"output_hidden_states",val:": Optional = None"},{name:"return_dict",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.VipLlavaForConditionalGeneration.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>) &#x2014;
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.</p>
<p>Indices can be obtained using <a href="/docs/transformers/pr_30929/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_30929/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/pr_30929/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.pixel_values",description:'<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, image_size, image_size)) -- The tensors corresponding to the input images. Pixel values can be obtained using [AutoImageProcessor](/docs/transformers/pr_30929/en/model_doc/auto#transformers.AutoImageProcessor). See [CLIPImageProcessor.__call__()](/docs/transformers/pr_30929/en/model_doc/imagegpt#transformers.ImageGPTFeatureExtractor.__call__) for details ([]</code>LlavaProcessor`] uses\n<a href="/docs/transformers/pr_30929/en/model_doc/clip#transformers.CLIPImageProcessor">CLIPImageProcessor</a> for processing images).',name:"pixel_values"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a></p>
<p>Indices can be obtained using <a href="/docs/transformers/pr_30929/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/pr_30929/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/pr_30929/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p>If <code>past_key_values</code> is used, optionally only the last <code>decoder_input_ids</code> have to be input (see
<code>past_key_values</code>).</p>
<p>If you want to change padding behavior, you should read <code>modeling_opt._prepare_decoder_attention_mask</code>
and modify to your needs. See diagram 1 in <a href="https://arxiv.org/abs/1910.13461" rel="nofollow">the paper</a> for more
information on the default strategy.</p>
<ul>
<li>1 indicates the head is <strong>not masked</strong>,</li>
<li>0 indicates the head is <strong>masked</strong>.</li>
</ul>`,name:"attention_mask"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>. <a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) &#x2014;
Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and 2 additional tensors of shape
<code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.VipLlavaForConditionalGeneration.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/pr_30929/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.</p>
<p>Args &#x2014;
labels (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>):
Labels for computing the masked language modeling loss. Indices should either be in <code>[0, ..., config.vocab_size]</code> or -100 (see <code>input_ids</code> docstring). Tokens with indices set to <code>-100</code> are ignored
(masked), the loss is only computed for the tokens with labels in <code>[0, ..., config.vocab_size]</code>.`,name:"return_dict"}],source:"https://github.com/huggingface/transformers/blob/vr_30929/src/transformers/models/vipllava/modeling_vipllava.py#L360",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>transformers.models.vipllava.modeling_vipllava.VipLlavaCausalLMOutputWithPast</code> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/pr_30929/en/model_doc/vipllava#transformers.VipLlavaConfig"
>VipLlavaConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Language modeling loss (for next-token prediction).</p>
</li>
<li>
<p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) — Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) — Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>)</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
<code>past_key_values</code> input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p>
</li>
<li>
<p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p>
</li>
<li>
<p><strong>image_hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the image embeddings, <code>(batch_size, num_images, sequence_length, hidden_size)</code>.</p>
<p>image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver</p>
</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>transformers.models.vipllava.modeling_vipllava.VipLlavaCausalLMOutputWithPast</code> or <code>tuple(torch.FloatTensor)</code></p>
`}}),V=new ot({props:{$$slots:{default:[rt]},$$scope:{ctx:k}}}),L=new Oe({props:{anchor:"transformers.VipLlavaForConditionalGeneration.forward.example",$$slots:{default:[lt]},$$scope:{ctx:k}}}),q=new st({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/vipllava.md"}}),{c(){s=l("meta"),w=i(),p=l("p"),c=i(),_(u.$$.fragment),o=i(),_(g.$$.fragment),se=i(),G=l("p"),G.innerHTML=We,ie=i(),Z=l("p"),Z.textContent=Ie,re=i(),F=l("p"),F.textContent=Be,le=i(),z=l("p"),z.innerHTML=Ne,de=i(),W=l("p"),W.textContent=He,ce=i(),I=l("ul"),I.innerHTML=Xe,pe=i(),_(B.$$.fragment),me=i(),N=l("p"),N.textContent=Pe,he=i(),_(H.$$.fragment),ue=i(),X=l("p"),X.innerHTML=Re,ge=i(),P=l("p"),P.innerHTML=Ye,fe=i(),_(R.$$.fragment),_e=i(),f=l("div"),_(Y.$$.fragment),Ce=i(),A=l("p"),A.innerHTML=Ee,Ue=i(),D=l("p"),D.innerHTML=Se,Je=i(),O=l("p"),O.innerHTML=Qe,$e=i(),_(x.$$.fragment),ve=i(),_(E.$$.fragment),be=i(),j=l("div"),_(S.$$.fragment),ke=i(),K=l("p"),K.innerHTML=qe,xe=i(),ee=l("p"),ee.innerHTML=Ae,Ve=i(),U=l("div"),_(Q.$$.fragment),Le=i(),te=l("p"),te.innerHTML=De,Ge=i(),_(V.$$.fragment),Ze=i(),_(L.$$.fragment),Me=i(),_(q.$$.fragment),ye=i(),ne=l("p"),this.h()},l(e){const t=at("svelte-u9bgzb",document.head);s=d(t,"META",{name:!0,content:!0}),t.forEach(n),w=r(e),p=d(e,"P",{}),ae(p).forEach(n),c=r(e),v(u.$$.fragment,e),o=r(e),v(g.$$.fragment,e),se=r(e),G=d(e,"P",{"data-svelte-h":!0}),h(G)!=="svelte-7p4sji"&&(G.innerHTML=We),ie=r(e),Z=d(e,"P",{"data-svelte-h":!0}),h(Z)!=="svelte-9bd7s8"&&(Z.textContent=Ie),re=r(e),F=d(e,"P",{"data-svelte-h":!0}),h(F)!=="svelte-vfdo9a"&&(F.textContent=Be),le=r(e),z=d(e,"P",{"data-svelte-h":!0}),h(z)!=="svelte-ym7i2j"&&(z.innerHTML=Ne),de=r(e),W=d(e,"P",{"data-svelte-h":!0}),h(W)!=="svelte-axv494"&&(W.textContent=He),ce=r(e),I=d(e,"UL",{"data-svelte-h":!0}),h(I)!=="svelte-t3cjqh"&&(I.innerHTML=Xe),pe=r(e),v(B.$$.fragment,e),me=r(e),N=d(e,"P",{"data-svelte-h":!0}),h(N)!=="svelte-mnlzle"&&(N.textContent=Pe),he=r(e),v(H.$$.fragment,e),ue=r(e),X=d(e,"P",{"data-svelte-h":!0}),h(X)!=="svelte-14nivh0"&&(X.innerHTML=Re),ge=r(e),P=d(e,"P",{"data-svelte-h":!0}),h(P)!=="svelte-1q33811"&&(P.innerHTML=Ye),fe=r(e),v(R.$$.fragment,e),_e=r(e),f=d(e,"DIV",{class:!0});var C=ae(f);v(Y.$$.fragment,C),Ce=r(C),A=d(C,"P",{"data-svelte-h":!0}),h(A)!=="svelte-1fh6dqb"&&(A.innerHTML=Ee),Ue=r(C),D=d(C,"P",{"data-svelte-h":!0}),h(D)!=="svelte-1cp2sn1"&&(D.innerHTML=Se),Je=r(C),O=d(C,"P",{"data-svelte-h":!0}),h(O)!=="svelte-o0f5yn"&&(O.innerHTML=Qe),$e=r(C),v(x.$$.fragment,C),C.forEach(n),ve=r(e),v(E.$$.fragment,e),be=r(e),j=d(e,"DIV",{class:!0});var J=ae(j);v(S.$$.fragment,J),ke=r(J),K=d(J,"P",{"data-svelte-h":!0}),h(K)!=="svelte-1jianjn"&&(K.innerHTML=qe),xe=r(J),ee=d(J,"P",{"data-svelte-h":!0}),h(ee)!=="svelte-hswkmf"&&(ee.innerHTML=Ae),Ve=r(J),U=d(J,"DIV",{class:!0});var $=ae(U);v(Q.$$.fragment,$),Le=r($),te=d($,"P",{"data-svelte-h":!0}),h(te)!=="svelte-1hemqrp"&&(te.innerHTML=De),Ge=r($),v(V.$$.fragment,$),Ze=r($),v(L.$$.fragment,$),$.forEach(n),J.forEach(n),Me=r(e),v(q.$$.fragment,e),ye=r(e),ne=d(e,"P",{}),ae(ne).forEach(n),this.h()},h(){oe(s,"name","hf:doc:metadata"),oe(s,"content",ct),oe(f,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),oe(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),oe(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){m(document.head,s),a(e,w,t),a(e,p,t),a(e,c,t),b(u,e,t),a(e,o,t),b(g,e,t),a(e,se,t),a(e,G,t),a(e,ie,t),a(e,Z,t),a(e,re,t),a(e,F,t),a(e,le,t),a(e,z,t),a(e,de,t),a(e,W,t),a(e,ce,t),a(e,I,t),a(e,pe,t),b(B,e,t),a(e,me,t),a(e,N,t),a(e,he,t),b(H,e,t),a(e,ue,t),a(e,X,t),a(e,ge,t),a(e,P,t),a(e,fe,t),b(R,e,t),a(e,_e,t),a(e,f,t),b(Y,f,null),m(f,Ce),m(f,A),m(f,Ue),m(f,D),m(f,Je),m(f,O),m(f,$e),b(x,f,null),a(e,ve,t),b(E,e,t),a(e,be,t),a(e,j,t),b(S,j,null),m(j,ke),m(j,K),m(j,xe),m(j,ee),m(j,Ve),m(j,U),b(Q,U,null),m(U,Le),m(U,te),m(U,Ge),b(V,U,null),m(U,Ze),b(L,U,null),a(e,Me,t),b(q,e,t),a(e,ye,t),a(e,ne,t),Te=!0},p(e,[t]){const C={};t&2&&(C.$$scope={dirty:t,ctx:e}),x.$set(C);const J={};t&2&&(J.$$scope={dirty:t,ctx:e}),V.$set(J);const $={};t&2&&($.$$scope={dirty:t,ctx:e}),L.$set($)},i(e){Te||(M(u.$$.fragment,e),M(g.$$.fragment,e),M(B.$$.fragment,e),M(H.$$.fragment,e),M(R.$$.fragment,e),M(Y.$$.fragment,e),M(x.$$.fragment,e),M(E.$$.fragment,e),M(S.$$.fragment,e),M(Q.$$.fragment,e),M(V.$$.fragment,e),M(L.$$.fragment,e),M(q.$$.fragment,e),Te=!0)},o(e){y(u.$$.fragment,e),y(g.$$.fragment,e),y(B.$$.fragment,e),y(H.$$.fragment,e),y(R.$$.fragment,e),y(Y.$$.fragment,e),y(x.$$.fragment,e),y(E.$$.fragment,e),y(S.$$.fragment,e),y(Q.$$.fragment,e),y(V.$$.fragment,e),y(L.$$.fragment,e),y(q.$$.fragment,e),Te=!1},d(e){e&&(n(w),n(p),n(c),n(o),n(se),n(G),n(ie),n(Z),n(re),n(F),n(le),n(z),n(de),n(W),n(ce),n(I),n(pe),n(me),n(N),n(he),n(ue),n(X),n(ge),n(P),n(fe),n(_e),n(f),n(ve),n(be),n(j),n(Me),n(ye),n(ne)),n(s),T(u,e),T(g,e),T(B,e),T(H,e),T(R,e),T(Y),T(x),T(E,e),T(S),T(Q),T(V),T(L),T(q,e)}}}const ct='{"title":"VipLlava","local":"vipllava","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"VipLlavaConfig","local":"transformers.VipLlavaConfig","sections":[],"depth":2},{"title":"VipLlavaForConditionalGeneration","local":"transformers.VipLlavaForConditionalGeneration","sections":[],"depth":2}],"depth":1}';function pt(k){return et(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class bt extends tt{constructor(s){super(),nt(this,s,pt,dt,Ke,{})}}export{bt as component};
