danielhanchen commited on
Commit
093fba6
·
verified ·
1 Parent(s): cf16ffd

Update config.json

Browse files
Files changed (1) hide show
  1. config.json +18 -8
config.json CHANGED
@@ -46,15 +46,25 @@
46
  "num_key_value_heads": 8,
47
  "num_local_experts": 32,
48
  "output_router_logits": false,
49
- "pad_token_id": 200017,
50
  "quantization_config": {
51
- "modules_to_not_convert": [
52
- "model.layers.*.self_attn",
53
- "model.layers.*.mlp.router",
54
- "model.embed_tokens",
55
- "lm_head"
 
 
 
 
 
 
 
56
  ],
57
- "quant_method": "mxfp4"
 
 
 
58
  },
59
  "rms_norm_eps": 1e-05,
60
  "rope_scaling": {
@@ -70,8 +80,8 @@
70
  "sliding_window": 128,
71
  "swiglu_limit": 7.0,
72
  "tie_word_embeddings": false,
 
73
  "transformers_version": "4.55.0",
74
- "unsloth_fixed": true,
75
  "use_cache": true,
76
  "vocab_size": 201088
77
  }
 
46
  "num_key_value_heads": 8,
47
  "num_local_experts": 32,
48
  "output_router_logits": false,
49
+ "pad_token_id": 199999,
50
  "quantization_config": {
51
+ "_load_in_4bit": true,
52
+ "_load_in_8bit": false,
53
+ "bnb_4bit_compute_dtype": "bfloat16",
54
+ "bnb_4bit_quant_storage": "uint8",
55
+ "bnb_4bit_quant_type": "nf4",
56
+ "bnb_4bit_use_double_quant": true,
57
+ "llm_int8_enable_fp32_cpu_offload": false,
58
+ "llm_int8_has_fp16_weight": false,
59
+ "llm_int8_skip_modules": [
60
+ "router",
61
+ "lm_head",
62
+ "embed_tokens"
63
  ],
64
+ "llm_int8_threshold": 6.0,
65
+ "load_in_4bit": true,
66
+ "load_in_8bit": false,
67
+ "quant_method": "bitsandbytes"
68
  },
69
  "rms_norm_eps": 1e-05,
70
  "rope_scaling": {
 
80
  "sliding_window": 128,
81
  "swiglu_limit": 7.0,
82
  "tie_word_embeddings": false,
83
+ "torch_dtype": "float16",
84
  "transformers_version": "4.55.0",
 
85
  "use_cache": true,
86
  "vocab_size": 201088
87
  }