Update configuration_gpt_refact.py

configuration_gpt_refact.py

@@ -1,7 +1,6 @@
 from transformers.configuration_utils import PretrainedConfig
 from transformers.utils import logging
 
-
 logger = logging.get_logger(__name__)
 
 
@@ -16,26 +15,23 @@ class GPTRefactConfig(PretrainedConfig):
     }
 
     def __init__(
-        self,
-        vocab_size: int = 49216,
-        n_positions: int = 4096,
-        n_embd: int = 1024,
-        n_layer: int = 32,
-        n_head: int = 64,
-        max_position_embeddings: int = 4096,
-        multi_query: bool = True,
-        layer_norm_epsilon=1e-5,
-        initializer_range=0.02,
-        scale_attn_weights=True,
-        use_cache=True,
-        bos_token_id=-1,
-        eos_token_id=0,
-        attention_softmax_in_fp32=False,
-        scale_attention_softmax_in_fp32=False,
-        resid_pdrop=0.1,
-        embd_pdrop=0.1,
-        attn_pdrop=0.1,
-        **kwargs,
+            self,
+            vocab_size: int = 49216,
+            n_positions: int = 4096,
+            n_embd: int = 1024,
+            n_layer: int = 32,
+            n_head: int = 64,
+            max_position_embeddings: int = 4096,
+            multi_query: bool = True,
+            layer_norm_epsilon: float = 1e-5,
+            initializer_range: float = 0.02,
+            use_cache: bool = True,
+            eos_token_id: int = 0,
+            attention_softmax_in_fp32: bool = True,
+            scale_attention_softmax_in_fp32: bool = True,
+            attention_bias_in_fp32: bool = True,
+            torch_dtype: str = 'bfloat16',
+            **kwargs,
     ):
         self.vocab_size = vocab_size
         self.n_positions = n_positions
@@ -43,19 +39,13 @@ class GPTRefactConfig(PretrainedConfig):
         self.n_layer = n_layer
         self.n_head = n_head
         self.n_inner = None
-        self.resid_pdrop = resid_pdrop
-        self.embd_pdrop = embd_pdrop
-        self.attn_pdrop = attn_pdrop
         self.layer_norm_epsilon = layer_norm_epsilon
         self.initializer_range = initializer_range
-        self.scale_attn_weights = scale_attn_weights
         self.use_cache = use_cache
         self.attention_softmax_in_fp32 = attention_softmax_in_fp32
         self.scale_attention_softmax_in_fp32 = scale_attention_softmax_in_fp32
-
-        self.bos_token_id = bos_token_id
-        self.eos_token_id = eos_token_id
-
+        self.attention_bias_in_fp32 = attention_bias_in_fp32
         self.multi_query = multi_query
         self.max_position_embeddings = max_position_embeddings
-        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
+        self.torch_dtype = torch_dtype
+        super().__init__(eos_token_id=eos_token_id, **kwargs)