Training in progress, step 40

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

00000000-0000-0000-0000-000000000000.yml

@@ -0,0 +1,50 @@
+base_model: peft-internal-testing/tiny-dummy-qwen2
+batch_size: 8
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- format: custom
+  path: argilla/databricks-dolly-15k-curated-en
+  type:
+    field_input: original-instruction
+    field_instruction: original-instruction
+    field_output: original-response
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/test-repo
+hub_strategy: checkpoint
+learning_rate: 0.002
+load_best_model_at_end: true
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 1
+micro_batch_size: 1
+model_type: AutoModelForCausalLM
+num_epochs: 100
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 8
+tokenizer_type: Qwen2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.001
+wandb_entity: ''
+wandb_mode: online
+wandb_name: peft-internal-testing/tiny-dummy-qwen2-argilla/databricks-dolly-15k-curated-en
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

03a659ff-e350-4bb9-8ff3-8c658a5d0dff.yml

@@ -0,0 +1,52 @@
+base_model: fxmarty/tiny-llama-fast-tokenizer
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - fc6136aac03f618a_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/fc6136aac03f618a_train_data.json
+  type:
+    field_instruction: text
+    field_output: title
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/b1c9c4ec-ffa2-429d-9c5b-90b5979c502d
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+special_tokens:
+  pad_token: </s>
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: fxmarty/tiny-llama-fast-tokenizer-/workspace/input_data/fc6136aac03f618a_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

077fd330-87f9-4bc4-b449-7713fbdaf1b0.yml

@@ -0,0 +1,51 @@
+base_model: unsloth/mistral-7b-v0.3
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - ca0152973425c947_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/ca0152973425c947_train_data.json
+  type:
+    field_input: code
+    field_instruction: func_name
+    field_output: docstring
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/5a2f5ce6-446b-4282-bb4d-9ee4e970231f
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/mistral-7b-v0.3-/tmp/ca0152973425c947_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

21315ae5-16ee-43cd-9612-743524060933.yml

@@ -0,0 +1,50 @@
+base_model: unsloth/Meta-Llama-3.1-8B
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 562fa3aeea07046a_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/562fa3aeea07046a_train_data.json
+  type:
+    field_instruction: prompt
+    field_output: text
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/c4596edc-efad-4776-86a1-caa06bffcada
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: PreTrainedTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/Meta-Llama-3.1-8B-/workspace/input_data/562fa3aeea07046a_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

2eaa630f-7785-4ca3-b46f-be41dcf74f78.yml

@@ -0,0 +1,51 @@
+base_model: katuni4ka/tiny-random-qwen1.5-moe
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 95544452e61c7393_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/95544452e61c7393_train_data.json
+  type:
+    field_input: input
+    field_instruction: instruction
+    field_output: output
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/e61e89f0-854a-4922-8d25-dae435e91af0
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: Qwen2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: katuni4ka/tiny-random-qwen1.5-moe-/workspace/input_data/95544452e61c7393_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

40f27435-f59d-488f-b2d6-01e356d79c48.yml

@@ -0,0 +1,50 @@
+base_model: Qwen/Qwen2-1.5B-Instruct
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - df925134bb2c32b8_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/df925134bb2c32b8_train_data.json
+  type:
+    field_instruction: prompt
+    field_output: amoral
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/ba640bbe-3257-40d8-88fe-26152f412bb7
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: Qwen2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: Qwen/Qwen2-1.5B-Instruct-/tmp/df925134bb2c32b8_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

427d02be-6008-4556-9a5e-9c7cb7503058.yml

@@ -0,0 +1,51 @@
+base_model: unsloth/Phi-3.5-mini-instruct
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 7e5b54272524b996_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/7e5b54272524b996_train_data.json
+  type:
+    field_input: input
+    field_instruction: instruction
+    field_output: output
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/ae26a9e9-089e-4d4a-b592-d8935df7c18d
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/Phi-3.5-mini-instruct-/workspace/input_data/7e5b54272524b996_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

54c39bbc-809b-4c67-a254-0e03a4884b4e.yml

@@ -0,0 +1,51 @@
+base_model: unsloth/codegemma-7b-it
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 2ebe89763cb3150d_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/2ebe89763cb3150d_train_data.json
+  type:
+    field_input: input
+    field_instruction: instruction
+    field_output: output
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/a0bc38f8-fcd3-4d7e-9a3f-3aa2e8a4204f
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: GemmaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/codegemma-7b-it-/tmp/2ebe89763cb3150d_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

5ff7bf5f-96dc-43dd-aeeb-560c0ab78db8.yml

@@ -0,0 +1,51 @@
+base_model: NousResearch/Hermes-3-Llama-3.1-8B
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 69447058613b41d8_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/69447058613b41d8_train_data.json
+  type:
+    field_input: sectionParentTitre
+    field_instruction: title_main
+    field_output: texte
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/942aa5fc-b540-46ce-b482-e38c4f637264
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: PreTrainedTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: NousResearch/Hermes-3-Llama-3.1-8B-/workspace/input_data/69447058613b41d8_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

63345f8a-4ec9-47f0-9956-6eaa52b2c2a6.yml

@@ -0,0 +1,52 @@
+base_model: heegyu/WizardVicuna-open-llama-3b-v2
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - fe9267419ea75ad2_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/fe9267419ea75ad2_train_data.json
+  type:
+    field_instruction: ca_topic
+    field_output: article
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/7114c34f-852f-43da-b985-b7f0b6d6d724
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+special_tokens:
+  pad_token: </s>
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: heegyu/WizardVicuna-open-llama-3b-v2-/tmp/fe9267419ea75ad2_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

6c7ae056-3b4d-460b-ba7b-a4000f32b3f1.yml

@@ -0,0 +1,51 @@
+base_model: unsloth/gemma-2-2b
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - b98d5b59c20c6595_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/b98d5b59c20c6595_train_data.json
+  type:
+    field_input: metadata
+    field_instruction: text
+    field_output: tags_str
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/429ee307-6dd2-4dd7-9e1d-7384d807a3df
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: GemmaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/gemma-2-2b-/tmp/b98d5b59c20c6595_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

75b21ca4-feab-4bdd-92b0-ea6d90dfa18f.yml

@@ -0,0 +1,51 @@
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - c6adcdcb593a3ee4_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/c6adcdcb593a3ee4_train_data.json
+  type:
+    field_input: abstract
+    field_instruction: question_en_origin
+    field_output: answer_en_origin
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/2faf844e-4a0a-4d23-95f4-a055e4864133
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: Qwen2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: Qwen/Qwen2.5-1.5B-Instruct-/workspace/input_data/c6adcdcb593a3ee4_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

879db250-c3f5-4d43-a7c5-c5a456ae5803.yml

@@ -0,0 +1,50 @@
+base_model: unsloth/Qwen2.5-Coder-1.5B-Instruct
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 4d85b564dafa38db_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/4d85b564dafa38db_train_data.json
+  type:
+    field_instruction: prompt
+    field_output: response
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/876ff803-5357-4240-8766-c54166515403
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: Qwen2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/Qwen2.5-Coder-1.5B-Instruct-/workspace/input_data/4d85b564dafa38db_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

8910478d-79cf-499e-8fed-7a2142f7ee60.yml

@@ -0,0 +1,51 @@
+base_model: unsloth/Phi-3-medium-4k-instruct
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - f6199f34ade98809_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/f6199f34ade98809_train_data.json
+  type:
+    field_input: choices
+    field_instruction: question
+    field_output: answer
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/2d37ba50-cd70-4895-be62-3477f5193e86
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/Phi-3-medium-4k-instruct-/tmp/f6199f34ade98809_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

README.md

@@ -0,0 +1,122 @@
+---
+library_name: transformers
+base_model: fxmarty/tiny-llama-fast-tokenizer
+tags:
+- axolotl
+- generated_from_trainer
+model-index:
+- name: b1c9c4ec-ffa2-429d-9c5b-90b5979c502d
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.4.1`
+```yaml
+base_model: fxmarty/tiny-llama-fast-tokenizer
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - fc6136aac03f618a_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/fc6136aac03f618a_train_data.json
+  type:
+    field_instruction: text
+    field_output: title
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/b1c9c4ec-ffa2-429d-9c5b-90b5979c502d
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+special_tokens:
+  pad_token: </s>
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: fxmarty/tiny-llama-fast-tokenizer-/workspace/input_data/fc6136aac03f618a_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true
+
+```
+
+</details><br>
+
+# b1c9c4ec-ffa2-429d-9c5b-90b5979c502d
+
+This model is a fine-tuned version of [fxmarty/tiny-llama-fast-tokenizer](https://huggingface.co/fxmarty/tiny-llama-fast-tokenizer) on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- training_steps: 18
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| No log        | 0.0071 | 1    | 10.3739         |
+
+
+### Framework versions
+
+- Transformers 4.46.0
+- Pytorch 2.5.0+cu124
+- Datasets 3.0.1
+- Tokenizers 0.20.1

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

config.json

@@ -0,0 +1,37 @@
+{
+  "_name_or_path": "katuni4ka/tiny-random-qwen1.5-moe",
+  "architectures": [
+    "Qwen2MoeForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "decoder_sparse_step": 1,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 32,
+  "initializer_range": 0.02,
+  "intermediate_size": 22,
+  "max_position_embeddings": 2048,
+  "max_window_layers": 2,
+  "mlp_only_layers": [],
+  "model_type": "qwen2_moe",
+  "moe_intermediate_size": 44,
+  "norm_topk_prob": false,
+  "num_attention_heads": 4,
+  "num_experts": 8,
+  "num_experts_per_tok": 4,
+  "num_hidden_layers": 4,
+  "num_key_value_heads": 2,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.001,
+  "shared_expert_intermediate_size": 22,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

da9e44b3-e4fb-4905-9c7c-6b03aad6b593.yml

@@ -0,0 +1,51 @@
+base_model: unsloth/SmolLM2-360M-Instruct
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - f1ccd02a885008e6_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/f1ccd02a885008e6_train_data.json
+  type:
+    field_input: target
+    field_instruction: user
+    field_output: assistant
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/3da0a03a-adbb-42e3-8fd7-bd7c0b1d3e9f
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: GPT2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/SmolLM2-360M-Instruct-/tmp/f1ccd02a885008e6_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

ee62f35d-1a99-4f1c-a69c-c91bc444b71f.yml

@@ -0,0 +1,53 @@
+base_model: EleutherAI/pythia-1b
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - b2a4966d9a5c880e_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/b2a4966d9a5c880e_train_data.json
+  type:
+    field_input: input
+    field_instruction: instruction
+    field_output: output
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/ee937811-31d0-4e11-944a-f4f8e06309d2
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+special_tokens:
+  pad_token: <|endoftext|>
+tokenizer_type: GPTNeoXTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: EleutherAI/pythia-1b-/workspace/input_data/b2a4966d9a5c880e_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

ef61f40b-eca8-4670-964b-fdd3d1d0f066.yml

@@ -0,0 +1,51 @@
+base_model: unsloth/SmolLM-135M
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 658988857b0a29c9_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/658988857b0a29c9_train_data.json
+  type:
+    field_input: choices
+    field_instruction: subject
+    field_output: question
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/09370687-f28e-45e5-91f6-f87011850a94
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: GPT2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/SmolLM-135M-/workspace/input_data/658988857b0a29c9_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "do_sample": true,
+  "eos_token_id": 1,
+  "pad_token_id": 1,
+  "transformers_version": "4.46.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14e6134b1d643c9c7f346b655687dc075b0fd0c7aa1ee4ce46f59192017c3ee2
+size 19782560

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ecbabedee28483af8dce99f4dd8fe36ef9c6c66877e669db930fe3569128330
+size 2071661

special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcfe42da0a4497e8b2b172c1f9f4ec423a46dc12907f4349c55025f670422ba9
+size 11418266

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "use_fast": true
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75877ad3818f65eedd53378ad3c43f71fa3e0c734e1e9a0f2b80517730b5a861
+size 6648