|
|
--- |
|
|
tags: |
|
|
- roleplay |
|
|
- creative-writing |
|
|
- merge |
|
|
- mergekit |
|
|
- kto |
|
|
- rl |
|
|
- finetune |
|
|
datasets: |
|
|
- Delta-Vector/Tauri-IFeval-Dans-Tulu-KTO |
|
|
- Delta-Vector/Tauri-Opus-Accepted-GPT-Rejected-Opus-Writing-Prompts |
|
|
- Delta-Vector/Tauri-KTO-Instruct-Mix |
|
|
- Delta-Vector/Tauri-Purpura-Arkhaios-CC-KTO |
|
|
base_model: |
|
|
- Delta-Vector/Archaeo-32B |
|
|
pipeline_tag: text-generation |
|
|
library_name: transformers |
|
|
--- |
|
|
``` |
|
|
__~a~_ |
|
|
~~; ~_ |
|
|
_ ~ ~_ _ |
|
|
'_\;__._._._._._._] ~_._._._._._.__;/_` |
|
|
'(/'/'/'/'|'|'|'| ( )|'|'|'|'\'\'\'\)' |
|
|
(/ / / /, | | | |(/ \) | | | ,\ \ \ \) |
|
|
(/ / / / / | | | ~(/ \) ~ | | \ \ \ \ \) |
|
|
(/ / / / / ~ ~ ~ (/ \) ~ ~ \ \ \ \ \) |
|
|
(/ / / / ~ / (||)| ~ \ \ \ \) |
|
|
~ / / ~ M /||\M ~ \ \ ~ |
|
|
~ ~ /||\ ~ ~ |
|
|
//||\\ |
|
|
//||\\ |
|
|
//||\\ |
|
|
'/||\' "Archaeopteryx" |
|
|
``` |
|
|
|
|
|
<html lang="en"> |
|
|
<head> |
|
|
<meta charset="UTF-8"> |
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0"> |
|
|
<style> |
|
|
@import url('https://fonts.googleapis.com/css2?family=VT323&display=swap'); |
|
|
body { |
|
|
background: #0a0017; |
|
|
margin: 0; |
|
|
padding: 20px; |
|
|
font-family: 'VT323', monospace; |
|
|
color: #ff00aa; |
|
|
text-shadow: 0 0 8px #ff00aa; |
|
|
animation: glitch-flicker 0.2s infinite alternate; |
|
|
} |
|
|
@keyframes glitch-flicker { |
|
|
0% { text-shadow: 0 0 5px #ff00aa, 0 0 15px #ff00aa; } |
|
|
100% { text-shadow: 0 0 8px #ff0066, 0 0 18px #ff0066; } |
|
|
} |
|
|
.crt-container { |
|
|
padding: 10px; |
|
|
max-width: 900px; |
|
|
margin: auto; |
|
|
} |
|
|
.crt-case { |
|
|
background: linear-gradient(135deg, #130021, #20002c); |
|
|
border-radius: 10px; |
|
|
padding: 15px; |
|
|
box-shadow: |
|
|
inset 2px 2px 10px rgba(255,0,170,0.5), |
|
|
2px 2px 5px rgba(255,0,170,0.3), |
|
|
0 0 25px rgba(255,0,170,0.2); |
|
|
} |
|
|
.crt-screen { |
|
|
background: #0c011a; |
|
|
padding: 20px; |
|
|
border-radius: 10px; |
|
|
box-shadow: |
|
|
inset 0 0 25px rgba(255,0,170,0.3), |
|
|
0 0 15px rgba(255,0,170,0.7); |
|
|
filter: contrast(1.2) brightness(1.2); |
|
|
text-shadow: 0px 0px 5px #ff00aa; |
|
|
animation: glow-pulse 3s infinite alternate; |
|
|
} |
|
|
@keyframes glow-pulse { |
|
|
0% { box-shadow: inset 0 0 20px rgba(255,0,170,0.3), 0 0 15px rgba(255,0,170,0.3); } |
|
|
100% { box-shadow: inset 0 0 30px rgba(255,0,170,0.5), 0 0 25px rgba(255,0,170,0.5); } |
|
|
} |
|
|
h2 { |
|
|
color: #ff33cc; |
|
|
text-align: center; |
|
|
font-size: 28px; |
|
|
text-shadow: |
|
|
0 0 8px #ff33cc, |
|
|
0 0 18px #ff0044; |
|
|
} |
|
|
pre { |
|
|
background: rgba(255,0,170,0.1); |
|
|
padding: 10px; |
|
|
border-radius: 10px; |
|
|
color: #ff66cc; |
|
|
font-size: 14px; |
|
|
box-shadow: inset 0 0 10px rgba(255,0,170,0.5); |
|
|
} |
|
|
.glitch { |
|
|
animation: text-glitch 0.5s infinite alternate; |
|
|
} |
|
|
@keyframes text-glitch { |
|
|
0% { transform: translateX(-2px); text-shadow: 0 0 5px #ff0066, 0 0 10px #ff33cc; } |
|
|
100% { transform: translateX(2px); text-shadow: 0 0 8px #ff00aa, 0 0 20px #ff0099; } |
|
|
} |
|
|
.neon-link { |
|
|
color: #ff66cc; |
|
|
text-decoration: none; |
|
|
transition: text-shadow 0.3s ease; |
|
|
} |
|
|
.neon-link:hover { |
|
|
text-shadow: 0px 0px 15px #ff66cc, 0 0 25px rgba(255,0,170,0.5); |
|
|
} |
|
|
.ascii-art { |
|
|
text-align: center; |
|
|
font-size: 12px; |
|
|
color: #ff33cc; |
|
|
text-shadow: 0px 0px 5px #ff00ff; |
|
|
margin-bottom: 20px; |
|
|
} |
|
|
.quantso-container { |
|
|
display: flex; |
|
|
justify-content: center; |
|
|
gap: 20px; |
|
|
margin-top: 20px; |
|
|
} |
|
|
.quantso-box { |
|
|
background: rgba(255,0,170,0.1); |
|
|
padding: 15px; |
|
|
border-radius: 10px; |
|
|
text-align: center; |
|
|
box-shadow: inset 0 0 10px rgba(255,0,170,0.5); |
|
|
flex: 1; |
|
|
max-width: 150px; |
|
|
} |
|
|
</style> |
|
|
</head> |
|
|
<body> |
|
|
<div class="crt-container"> |
|
|
<div class="crt-case"> |
|
|
<div class="crt-screen"> |
|
|
<p>Support me on Ko-Fi: https://ko-fi.com/deltavector</p> |
|
|
<p>A series of Merges made for Roleplaying & Creative Writing, This model is a RL train ontop of Archaeo. A merge using Hamanasu-Magnum & Kunou, Trained with Axolotl on 8xH200s.</p> |
|
|
<h3>ChatML formatting</h3> |
|
|
<pre> |
|
|
"""<|im_start|>system |
|
|
system prompt<|im_end|> |
|
|
<|im_start|>user |
|
|
Hi there!<|im_end|> |
|
|
<|im_start|>assistant |
|
|
Nice to meet you!<|im_end|> |
|
|
<|im_start|>user |
|
|
Can I ask a question?<|im_end|> |
|
|
<|im_start|>assistant |
|
|
""" |
|
|
</pre> |
|
|
<h3>Axolotl Configuration</h3> |
|
|
<pre> |
|
|
base_model: ./model |
|
|
|
|
|
plugins: |
|
|
- axolotl.integrations.liger.LigerPlugin |
|
|
- axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin |
|
|
liger_rope: true |
|
|
liger_rms_norm: true |
|
|
liger_layer_norm: true |
|
|
liger_glu_activation: true |
|
|
liger_fused_linear_cross_entropy: true |
|
|
cut_cross_entropy: false |
|
|
|
|
|
load_in_8bit: false |
|
|
load_in_4bit: false |
|
|
strict: false |
|
|
|
|
|
rl: kto |
|
|
kto_undesirable_weight: 1.0 |
|
|
|
|
|
datasets: |
|
|
- path: Delta-Vector/Tauri-Opus-Accepted-GPT-Rejected-Opus-Writing-Prompts |
|
|
split: train |
|
|
type: chatml.argilla |
|
|
- path: Delta-Vector/Tauri-IFeval-Dans-Tulu-KTO |
|
|
split: train |
|
|
type: chatml.argilla |
|
|
- path: Delta-Vector/Tauri-KTO-Instruct-Mix |
|
|
split: train |
|
|
type: chatml.argilla |
|
|
- path: Delta-Vector/Tauri-Purpura-Arkhaios-CC-KTO |
|
|
split: train |
|
|
type: chatml.argilla |
|
|
dataset_prepared_path: last_run_prepared |
|
|
val_set_size: 0.0 |
|
|
output_dir: ./archaeo-kto-v2 |
|
|
remove_unused_columns: false |
|
|
|
|
|
#@lora_mlp_kernel: true |
|
|
#lora_qkv_kernel: true |
|
|
#lora_o_kernel: true |
|
|
|
|
|
|
|
|
adapter: lora |
|
|
lora_model_dir: |
|
|
|
|
|
sequence_len: 8192 |
|
|
pad_to_sequence_len: false |
|
|
|
|
|
lora_r: 64 |
|
|
lora_alpha: 32 |
|
|
lora_dropout: 0.0 |
|
|
lora_target_linear: true |
|
|
lora_fan_in_fan_out: |
|
|
lora_target_modules: |
|
|
- gate_proj |
|
|
- down_proj |
|
|
- up_proj |
|
|
- q_proj |
|
|
- v_proj |
|
|
- k_proj |
|
|
- o_proj |
|
|
|
|
|
wandb_project: Francois-V2 |
|
|
wandb_entity: |
|
|
wandb_watch: |
|
|
wandb_name: Archaeo-32b-KTO |
|
|
wandb_log_model: |
|
|
|
|
|
gradient_accumulation_steps: 4 |
|
|
micro_batch_size: 4 |
|
|
num_epochs: 1 |
|
|
optimizer: paged_ademamix_8bit |
|
|
lr_scheduler: constant_with_warmup |
|
|
learning_rate: 5e-6 |
|
|
max_grad_norm: 0.001 |
|
|
|
|
|
train_on_inputs: false |
|
|
group_by_length: false |
|
|
bf16: auto |
|
|
fp16: |
|
|
tf32: true |
|
|
|
|
|
gradient_checkpointing: true |
|
|
gradient_checkpointing_kwargs: |
|
|
use_reentrant: true |
|
|
early_stopping_patience: |
|
|
resume_from_checkpoint: |
|
|
local_rank: |
|
|
logging_steps: 1 |
|
|
xformers_attention: |
|
|
flash_attention: true |
|
|
|
|
|
warmup_steps: 100 |
|
|
evals_per_epoch: 4 |
|
|
eval_table_size: |
|
|
eval_max_new_tokens: 128 |
|
|
saves_per_epoch: 1 |
|
|
debug: |
|
|
deepspeed: ./deepspeed_configs/zero3_bf16.json |
|
|
weight_decay: 0.0025 |
|
|
fsdp: |
|
|
fsdp_config: |
|
|
</pre> |
|
|
<h3>Quants:</h3> |
|
|
<div class="quantso-container"> |
|
|
<div class="quantso-box"> |
|
|
<strong>GGUF</strong><br> |
|
|
<a class="neon-link" href="#">https://huggingface.co/mradermacher/Axo-Merge-Archaeo-V2-Lora-GGUF</a> |
|
|
</div> |
|
|
<div class="quantso-box"> |
|
|
<strong>EXL2</strong><br> |
|
|
<a class="neon-link" href="#">Delta-Vector/Archaeo-32B-KTO-exl2</a> |
|
|
</div> |
|
|
</div> |
|
|
<h3>Credits</h3> |
|
|
<p>Thank you to: Kubernetes-bad, LucyKnada, Kalomaze, Alicat, Intervitens, Samantha Twinkman, Tav, Trappu & The rest of Anthracite</p> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
</body> |
|
|
</html> |
