diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..52373fe24473b1aa44333d318f578ae6bf04b49b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..49949673313bbbb76583b3b860f56a5bba8539ee --- /dev/null +++ b/README.md @@ -0,0 +1,59 @@ +## Precision: FP32 vs FP16 (and BF16) + +This project saves dequantized checkpoints in **FP16** (default) or **BF16**. +Here’s what those formats mean and when you might choose each: + +### TL;DR + +- **FP32 (single precision, 32-bit, 4 bytes/param)** + Reference/default precision in many frameworks. Highest numerical range/precision, **largest memory**. +- **FP16 (half precision, 16-bit, 2 bytes/param)** + Half the memory of FP32. Great for **inference** on modern GPUs; may underflow/overflow more easily than BF16. +- **BF16 (bfloat16, 16-bit, 2 bytes/param)** + Same memory as FP16, **wider exponent like FP32**, often more numerically robust than FP16; slightly less precision in mantissa. + +> In this repo, output precision is **FP16** (default) or **BF16** via `--dtype`. +> **FP32 output is not offered** because it doubles disk/RAM vs FP16/BF16 with minimal inference benefit on modern hardware. + +### Memory math (example: 120B parameters) + +Each parameter stores one number: + +| Format | Bits | Bytes/param | Approx size for 120B params | +|-------:|-----:|-------------:|-----------------------------:| +| FP32 | 32 | 4 | ~ **447 GiB** | +| FP16 | 16 | 2 | ~ **224 GiB** | +| BF16 | 16 | 2 | ~ **224 GiB** | + +> Calculation (GiB): `params * bytes_per_param / 1024^3` +> For 120,000,000,000 params: +> FP32: 480e9 B ≈ 447.03 GiB +> FP16/BF16: 240e9 B ≈ 223.52 GiB + +### When to use which + +- **Inference on modern NVIDIA GPUs (Turing+/Ampere+/Ada/Hopper):** + Use **FP16** (default here) or **BF16**. You’ll get large memory savings and typically **equal or faster** throughput than FP32 thanks to tensor cores. + +- **Training / Finetuning:** + Use **mixed precision** (BF16 or FP16 compute with an FP32 master copy of weights/optimizer states). + If your GPU supports BF16 well (e.g., A100/H100), **BF16** is preferred for numeric stability. + (This tool focuses on exporting dequantized checkpoints, not training loops.) + +- **If you hit numeric issues in FP16:** + Try **BF16** (`--dtype bf16`). Same size as FP16 but usually more stable due to FP32-like exponent range. + +### Notes + +- **FP32** remains the gold standard for numeric headroom and deterministic baselines, but for **inference** it’s typically unnecessary and **costly** (2× memory vs FP16/BF16). +- **Tensor cores** accelerate FP16/BF16 GEMMs on most modern NVIDIA GPUs; FP32 is often slower and more memory-bound. +- If a downstream runtime expects a specific dtype, export to that: FP16 for speed/memory, BF16 for robustness. + +--- + +### WIP + +- Upcoming models: cleaned FP16 release (uniform fp16 with fp32 LayerNorms), compressed variants (W8A8, W4A16, mixed experts), and smaller distilled checkpoints. +- Evals: MMLU, HellaSwag, TruthfulQA, GSM8K, BBH, MT‑Bench; plus latency/throughput and memory footprint on 3090/A100. +- Extras: scripted upload tooling, detailed model cards, and reproducible Docker workflows. + diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8db2ec0dd427e1922f6923024816c244d209e2db --- /dev/null +++ b/config.json @@ -0,0 +1,135 @@ +{ + "vocab_size": 201088, + "hidden_size": 2880, + "intermediate_size": 2880, + "num_hidden_layers": 36, + "num_attention_heads": 64, + "num_local_experts": 128, + "sliding_window": 128, + "num_experts_per_tok": 4, + "num_key_value_heads": 8, + "hidden_act": "silu", + "initializer_range": 0.02, + "rms_norm_eps": 0.00001, + "rope_theta": 150000, + "rope_scaling": { + "beta_fast": 32.0, + "beta_slow": 1.0, + "factor": 32.0, + "original_max_position_embeddings": 4096, + "rope_type": "yarn", + "truncate": false + }, + "attention_dropout": 0.0, + "head_dim": 64, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "attention_bias": true, + "max_position_embeddings": 131072, + "router_aux_loss_coef": 0.9, + "output_router_logits": false, + "use_cache": true, + "return_dict": true, + "output_hidden_states": false, + "torchscript": false, + "torch_dtype": "float16", + "pruned_heads": {}, + "tie_word_embeddings": false, + "chunk_size_feed_forward": 0, + "is_encoder_decoder": false, + "is_decoder": false, + "cross_attention_hidden_size": null, + "add_cross_attention": false, + "tie_encoder_decoder": false, + "architectures": [ + "GptOssForCausalLM" + ], + "finetuning_task": null, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "task_specific_params": null, + "problem_type": null, + "tokenizer_class": null, + "prefix": null, + "bos_token_id": null, + "pad_token_id": 199999, + "eos_token_id": 200002, + "sep_token_id": null, + "decoder_start_token_id": null, + "max_length": 20, + "min_length": 0, + "do_sample": false, + "early_stopping": false, + "num_beams": 1, + "num_beam_groups": 1, + "diversity_penalty": 0.0, + "temperature": 1.0, + "top_k": 50, + "top_p": 1.0, + "typical_p": 1.0, + "repetition_penalty": 1.0, + "length_penalty": 1.0, + "no_repeat_ngram_size": 0, + "encoder_no_repeat_ngram_size": 0, + "bad_words_ids": null, + "num_return_sequences": 1, + "output_scores": false, + "return_dict_in_generate": false, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "remove_invalid_values": false, + "exponential_decay_length_penalty": null, + "suppress_tokens": null, + "begin_suppress_tokens": null, + "_name_or_path": "openai/gpt-oss-120b", + "transformers_version": "4.55.1", + "experts_per_token": 4, + "initial_context_length": 4096, + "model_type": "gpt_oss", + "swiglu_limit": 7.0, + "tf_legacy_loss": false, + "use_bfloat16": false, + "output_attentions": false +} diff --git a/model-00001-of-00073.safetensors b/model-00001-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..febf6056ffb0fce1868d912a370c7e1c08067aba --- /dev/null +++ b/model-00001-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d2cbeb91ab681dacd107aaabeebdaa7ce658189f486954f66c91be68ed51eae +size 1212106024 diff --git a/model-00002-of-00073.safetensors b/model-00002-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ac5fa8df37e4d7ce7b5a2d97b7067135f3da6bb7 --- /dev/null +++ b/model-00002-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a106d6a1cbfedad04ba58fe55341c5d0662479348151c683f70152ed5b7295f +size 4248207608 diff --git a/model-00003-of-00073.safetensors b/model-00003-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f7ef77718c49f421b513adf77c3a206c27c367a5 --- /dev/null +++ b/model-00003-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20e43143de2a41a7f5b4ac3e16fb94ba45a7bfbbd45ba4ed9e838779010caf79 +size 2177966200 diff --git a/model-00004-of-00073.safetensors b/model-00004-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ef276df73dd403146c9e7a06273a2d7e12094093 --- /dev/null +++ b/model-00004-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f490a24b4ef54c1b81da8ca86689e9962956cebaca6e585a08d80fd435d5c7e5 +size 4248207608 diff --git a/model-00005-of-00073.safetensors b/model-00005-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a44c9b2f9d7cfcb6ec4075ba74b4fc1de08af4d5 --- /dev/null +++ b/model-00005-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:000240ccc86960cfea8dc7c52bf82375b4b07c22201d5c0dc0c184e84387ee80 +size 2177966200 diff --git a/model-00006-of-00073.safetensors b/model-00006-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2da2c3fb57c614e380a599ba337c1790df0d9a99 --- /dev/null +++ b/model-00006-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:746c2890ddbb9d176d6660916ba68d37bde3eca64cb203f7a28e539dd354a078 +size 4248207608 diff --git a/model-00007-of-00073.safetensors b/model-00007-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..909ea461c7eef8a60e3cc6d41656a8896f3905db --- /dev/null +++ b/model-00007-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe87b82b49dd8101577e5a68927a0a87f0b8811562362bc1311d0739eec047da +size 2177966200 diff --git a/model-00008-of-00073.safetensors b/model-00008-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a665d51d5f15329e8deaae792edb99e7fea4bab8 --- /dev/null +++ b/model-00008-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:384dd1a38790cf44ace59438fd4ffdeebdfa90d0025b936cf99039c33de0fda2 +size 4248207608 diff --git a/model-00009-of-00073.safetensors b/model-00009-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2fe7eddc3f7840a7f73da9b9c51f84beb2046426 --- /dev/null +++ b/model-00009-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:168709ddd68dfeda0827e9a10d4e5f57505f23f4886693ddf6174f6de7106493 +size 2177966200 diff --git a/model-00010-of-00073.safetensors b/model-00010-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5158cf8b89bc749667cd44fe368e638cea5b6c99 --- /dev/null +++ b/model-00010-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0389fca580334c41f6ab048cc8a89dc43ae4bf3b69d8498d86c443af5611938 +size 4248207608 diff --git a/model-00011-of-00073.safetensors b/model-00011-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b39ebb6e8d8e1a023583b4c5cd5a57748371306d --- /dev/null +++ b/model-00011-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:893b5dbb553a442a1bbb0ddfe86820d94b9a02bdc6d2df8ce0f05a21947f54f4 +size 2177966200 diff --git a/model-00012-of-00073.safetensors b/model-00012-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..81949330fa4e1026d79a65a57fb92dc0cb0f7b1b --- /dev/null +++ b/model-00012-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c27f5622cf1c5f2264f278c2f06653fd9acdbb04c01b1956e44317a1c9755c3 +size 4248207608 diff --git a/model-00013-of-00073.safetensors b/model-00013-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ec91d59e4fc9052315ac6eb99a4d7228a74888b4 --- /dev/null +++ b/model-00013-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c909b19ef07269c000f667cdea42df693a04fbdf46e3b724285dd6385963b14 +size 2177966200 diff --git a/model-00014-of-00073.safetensors b/model-00014-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1a98c59300abbb5fdb9c89be24cc5657ea874e59 --- /dev/null +++ b/model-00014-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95f5f32613862b01373a5b0fb79c828db876e53a06a396f5beac60ed3e0cd789 +size 4248207608 diff --git a/model-00015-of-00073.safetensors b/model-00015-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dce51d293888be16ae38118eb2659fcb7d47044c --- /dev/null +++ b/model-00015-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e57df02381f6981b212c17cf6db3540ab61df6d9d5d44ac1d25af70b4b1cf373 +size 2177966200 diff --git a/model-00016-of-00073.safetensors b/model-00016-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..87182ea1312bd62fd04ae25a5714f0bdf82176bf --- /dev/null +++ b/model-00016-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:882418dc94e57875150aa6f0dda3bc039b9cec42bc693ce540c483d9ce9d0089 +size 4248207608 diff --git a/model-00017-of-00073.safetensors b/model-00017-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b812f029ced2c2b392b1ed7d62692b94241ff3cb --- /dev/null +++ b/model-00017-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee6e015481ae64c8863b9ec7b5f6e88a93c5758d5971f3850dfaadbbce065e59 +size 2177966200 diff --git a/model-00018-of-00073.safetensors b/model-00018-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0261a5a16fa26d9ca1b02d9b0e9434645ab8fc48 --- /dev/null +++ b/model-00018-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:209b8e4122ca03062ba1134d1763f7dfe4c7e2d6a85a0fcda1d8700ebeb58c1b +size 4248207608 diff --git a/model-00019-of-00073.safetensors b/model-00019-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3a939659ed33855d3cf6904bf29cc331da1ae36b --- /dev/null +++ b/model-00019-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6dd629c13b37f3a8ac58dbe4c5b00ce85e0b3c4e70197f0ac7ca8092e5e33f3 +size 2177966200 diff --git a/model-00020-of-00073.safetensors b/model-00020-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4c11d742f036308ae80892b0c0b383dcc43b4c44 --- /dev/null +++ b/model-00020-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a08672de7209f0a375991ee4ca9cfb89817e4ae309b510190e5a556fc825db44 +size 4248207608 diff --git a/model-00021-of-00073.safetensors b/model-00021-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..36ee85f0160216bfecbd55d6b5cae4f6b7924e04 --- /dev/null +++ b/model-00021-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a92acdaf093342d2ac369b872b1b83dd9de4aa25f777d0d11a82494a709113d +size 2177966152 diff --git a/model-00022-of-00073.safetensors b/model-00022-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4a210d7116017877ab3da63636a7681c7df32fa8 --- /dev/null +++ b/model-00022-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6280c4f71600f0d8464c5372123ec5ba0e44c97ac688fa533e609a100eef35db +size 4248207608 diff --git a/model-00023-of-00073.safetensors b/model-00023-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a9aec3c633b7c1c9d62608864d856fc5815413dd --- /dev/null +++ b/model-00023-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fef9ded36a25b51c66bdadde514ba58cb133abc91ee71506c91fd3b2f861229 +size 2177966216 diff --git a/model-00024-of-00073.safetensors b/model-00024-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3075a6044d18d096e717424fdbc4f29e854df816 --- /dev/null +++ b/model-00024-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1cfb393370bf9259f926ecd35a9f384da00b994380d8817f3105dcd57747f8b +size 4248207608 diff --git a/model-00025-of-00073.safetensors b/model-00025-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b99647da69ac88e4743235c52f4536223786109e --- /dev/null +++ b/model-00025-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7c17e9181eae9c67dfc7549e9496a0e2e671be8cb6d3de025927238c95daeef +size 2177966216 diff --git a/model-00026-of-00073.safetensors b/model-00026-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..63fd1ace28096c97b36a671c1e7185f0164e3ec4 --- /dev/null +++ b/model-00026-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d42e377e00438cc5f12ade59fc5e3effba7dfcaf5b18e9653f90da670a0d1af +size 4248207608 diff --git a/model-00027-of-00073.safetensors b/model-00027-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8b2e29da7f7addb7df596c7e427bd592c55d8aa0 --- /dev/null +++ b/model-00027-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50b154ebc25911c44468f2f44961ea7c559cadb5c5feb8f5575d16c2517431d1 +size 2177966216 diff --git a/model-00028-of-00073.safetensors b/model-00028-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..854a32a3d1237fbd62c60587c8fa65a66aa64226 --- /dev/null +++ b/model-00028-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5314c859821cc46eda633aca96bae5d512eb35bfbd7d25b157c971e04e52ca69 +size 4248207608 diff --git a/model-00029-of-00073.safetensors b/model-00029-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..635c43c33fc1adad16b62965c48f4d8908843f8f --- /dev/null +++ b/model-00029-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a32ed91720cda3ec9c7a7fcf27ce78e4225b15fa50430a0f788808234d0775c2 +size 2177966216 diff --git a/model-00030-of-00073.safetensors b/model-00030-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb65dcbaf83f62436c3e7562d67f7a2b3d6f3dbb --- /dev/null +++ b/model-00030-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d06097e01bb51703a688f1e86838779903ba117a104810da64ced95fe462af72 +size 4248207608 diff --git a/model-00031-of-00073.safetensors b/model-00031-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4a97a6cda38fd525548124db1e56155e6e492a04 --- /dev/null +++ b/model-00031-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0ac34e3e886e0f05ceaab5864b8ea5b97629c2e9d76fe6ca946ef01694df6b8 +size 2177966216 diff --git a/model-00032-of-00073.safetensors b/model-00032-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ef50cce9b63c7ecc2754590b7cdd391aa8c00af4 --- /dev/null +++ b/model-00032-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e16fbe7fddd34ca5f8270f627be113d8c4eb53db14ba238b2128fa4c9b3addbd +size 4248207608 diff --git a/model-00033-of-00073.safetensors b/model-00033-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c3b3dc8fbed08200a5974f8c0b7476e2147daa4c --- /dev/null +++ b/model-00033-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08f466aa0f45c1680fe817ef426e5dfcbdb6a49dbf88cc444dd1c49482967fda +size 2177966216 diff --git a/model-00034-of-00073.safetensors b/model-00034-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b57aff735e410c90840bb56342441caecc272b27 --- /dev/null +++ b/model-00034-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da4efac69476ee725226941c7505cdec52a2be88603e1ec08f356e6a1ad228a0 +size 4248207608 diff --git a/model-00035-of-00073.safetensors b/model-00035-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8ca515de09e7e306018673c4143ffeec7a23d816 --- /dev/null +++ b/model-00035-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:325a31858596fd5b1ec971300c097d1ca0dfafff0953b2622739c4503456cbc7 +size 2177966216 diff --git a/model-00036-of-00073.safetensors b/model-00036-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5d4e39ebf96c7d4ce4f657d1382806a65ed6dad5 --- /dev/null +++ b/model-00036-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e0b23972d8a42c05557793a837ce46350b0bf3b1383110ebdaecc059e25b0bd +size 4248207608 diff --git a/model-00037-of-00073.safetensors b/model-00037-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d1fd4b8eb84251d90432a51b51e9ebd963e90d28 --- /dev/null +++ b/model-00037-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6784d0a32a398ac929e875b703d033f517f408415eb3705e6770b5320a2206b8 +size 2177966216 diff --git a/model-00038-of-00073.safetensors b/model-00038-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4fac381ec626655aeb17efb470f5cc9da3c61acf --- /dev/null +++ b/model-00038-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5088b51769655642be1d7521f6e054659a4a5dbeee75668ddd2f8fa2c63faf3 +size 4248207608 diff --git a/model-00039-of-00073.safetensors b/model-00039-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..97ed6abf3da49264e87db3533f01c363a8a97952 --- /dev/null +++ b/model-00039-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2638fd7664b8ebefdd33cc53f25a6d3aa283defbdf9ef649fd0f3739a1685e6b +size 2177966216 diff --git a/model-00040-of-00073.safetensors b/model-00040-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..274177efda997e7abbd7748f3c07af7b0f544775 --- /dev/null +++ b/model-00040-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:737c35d33842a328fec4a998c11eb041964dc65ff2da5887a088b36aefa1ebfd +size 4248207608 diff --git a/model-00041-of-00073.safetensors b/model-00041-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..21fe67d0a3ce40126ef51d2c6a7e33c25a860b3b --- /dev/null +++ b/model-00041-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a96999e1a529f0421389416aaf14d4857070a34e4764b5e4e3519325ec99e71 +size 2177966216 diff --git a/model-00042-of-00073.safetensors b/model-00042-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5677e4e3ceb6199f35461bda59bcdb69642939f4 --- /dev/null +++ b/model-00042-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0d719182ffb2b82beba426c2f56e6f8f8f854169ab189c45c2bc9fb2271556b +size 4248207608 diff --git a/model-00043-of-00073.safetensors b/model-00043-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5c0345155b3c0d730079cb448a006c241b8a8016 --- /dev/null +++ b/model-00043-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe1340d9628d1829ec086e1d21855b3d04e3e137fe44f9ba698b8cb6fa600091 +size 2177966216 diff --git a/model-00044-of-00073.safetensors b/model-00044-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4d20178a9f36fb0b098bc19375ae8fdc8ec25fe1 --- /dev/null +++ b/model-00044-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1ad8e6e5d06780f9c6a09e991a71f0978ba9e88b7224f146a5933c13f1f319c +size 4248207608 diff --git a/model-00045-of-00073.safetensors b/model-00045-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..641c45d66843275675e53f35548f7e405ea8ea6c --- /dev/null +++ b/model-00045-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21c53d8e8a7827388018fec44f0de55f7079493ef9a5ec697f2c3d3258cc1565 +size 2177966216 diff --git a/model-00046-of-00073.safetensors b/model-00046-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..be2073fafc3010edec5b4f6b2683c4a94043eaa2 --- /dev/null +++ b/model-00046-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3650773042b703b7b4974e3877f4eb25152f4e9b27500788357bc55651bc8c86 +size 4248207608 diff --git a/model-00047-of-00073.safetensors b/model-00047-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3415c63fc1795d8301026313a1c807c86dcbc908 --- /dev/null +++ b/model-00047-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54f3dcfa2f275c5271776797922415c966a944710f791f8c11788ebef072ec06 +size 2177966216 diff --git a/model-00048-of-00073.safetensors b/model-00048-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4503e7772aea62867c19b5348b42b6c70ad8ba4e --- /dev/null +++ b/model-00048-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:132065e22d6027264ba7cb2740707f21fc3116b526d402a65b73fa41d5c71c68 +size 4248207608 diff --git a/model-00049-of-00073.safetensors b/model-00049-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..88cf9016e061496341194f297ccace2cdb6ff17a --- /dev/null +++ b/model-00049-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e76137949def2f9c8354b4aa37ecd0ef95ce871c4808418db3a9dfbec739af13 +size 2177966216 diff --git a/model-00050-of-00073.safetensors b/model-00050-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..34b92ea0d3fff7f426ca49f8c7a05ac149d8f429 --- /dev/null +++ b/model-00050-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdc22fa94d23e8a483f09409dcfa62c5236759e36ebbe8419a2bd1432d31a0e9 +size 4248207608 diff --git a/model-00051-of-00073.safetensors b/model-00051-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d42b168364ed88298c85c88b7c5ff3acf344aca7 --- /dev/null +++ b/model-00051-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:906d2f5a4eced89810f17a87991ae2dd9622ab45174d0c577f91b25febd48872 +size 2177966216 diff --git a/model-00052-of-00073.safetensors b/model-00052-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8eb46dcb739a6d6f007b27c729e5d519359f79c5 --- /dev/null +++ b/model-00052-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6329fab595ebeb4f7e1c760cddd41db3ea5410bef1fa3995b3e532b7c07e2ff1 +size 4248207608 diff --git a/model-00053-of-00073.safetensors b/model-00053-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bbbb8cd0be398e6fd86a0eaa520a0c7c15bf957e --- /dev/null +++ b/model-00053-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26b5400cef8e6c518ead03ea214eda40c9830a99f8dcff9f04ba3a431633568d +size 2177966216 diff --git a/model-00054-of-00073.safetensors b/model-00054-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0fdc2863241fab3c5c79f6b7a352a5d2a8b754b2 --- /dev/null +++ b/model-00054-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5bdc21ac9739044a0c860f371602514b3584d8ed180eb09e6f92e945f882ea7 +size 4248207608 diff --git a/model-00055-of-00073.safetensors b/model-00055-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..90348f1ad7de38649683364ecd31b32f7d5e06c2 --- /dev/null +++ b/model-00055-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e74ca3cb2fca1b7241dcd60e17b9295fde828a4ededcafd2c5cbd9a638f340a9 +size 2177966216 diff --git a/model-00056-of-00073.safetensors b/model-00056-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ed06c6fe3a3aedcfe752bd9f5a8c9bd81f67f6f6 --- /dev/null +++ b/model-00056-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce8575e3c4eb1a479ab8182cf046a0d62b7275c3b4b742ca60133973e03cb9d0 +size 4248207608 diff --git a/model-00057-of-00073.safetensors b/model-00057-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..62c136d8e9966d9e28bb7c005c74f82cb6aebb00 --- /dev/null +++ b/model-00057-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b96513984239d015bb1dab5198a50d0add295846b61c71b0ae87eb9193570a7 +size 2177966216 diff --git a/model-00058-of-00073.safetensors b/model-00058-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0f7ac5a18f60cc69291e54a3e7cf70a5c0fd347c --- /dev/null +++ b/model-00058-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baf84eb13ef630bc6c2122387e7bb661d577181e61021e742317e5beaeb0811b +size 4248207608 diff --git a/model-00059-of-00073.safetensors b/model-00059-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9fb63236e8d53f044cb507a2327e9b28c68cab67 --- /dev/null +++ b/model-00059-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0bd1c8295886dc5d1f2e9bbbd76f03e3321341e480e2a72a9905bab22b667f5 +size 2177966216 diff --git a/model-00060-of-00073.safetensors b/model-00060-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e10afb7ee2e4e04a780e04d071f7fd560344e64d --- /dev/null +++ b/model-00060-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bec716b8dab398322007364dddaccd2af60824feec69db3c4782841a0c30ce9 +size 4248207608 diff --git a/model-00061-of-00073.safetensors b/model-00061-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8d449ecd0b04b20703f830f560c7a6a6f83270b2 --- /dev/null +++ b/model-00061-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8669657084f092175f230a7a7345a1a0bf38e58d586f98a482ff8e44094af3db +size 2177966216 diff --git a/model-00062-of-00073.safetensors b/model-00062-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..086f1ff46f27162c8c09d9670e42b16cce45c951 --- /dev/null +++ b/model-00062-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b41b3c504a290edd3ca58f15b435f237e352f3a0ad755b107ce40adb3db71a27 +size 4248207608 diff --git a/model-00063-of-00073.safetensors b/model-00063-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8953c6ae45be955e8472bb241e580c259daa9176 --- /dev/null +++ b/model-00063-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cbd96a31fd7b0a73d640c40b4486a7f9871ac2f8bdcff638d57c9da24c6898d +size 2177966216 diff --git a/model-00064-of-00073.safetensors b/model-00064-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1a44f40a86d38643db24d2e5b400dd83729873f1 --- /dev/null +++ b/model-00064-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fe027632aead645095abf41c2894d1cd756a57fb632f6a1a5d6ee0c1e0c776c +size 4248207608 diff --git a/model-00065-of-00073.safetensors b/model-00065-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ca600379e2623cf07034f6c98f9a181518a9440a --- /dev/null +++ b/model-00065-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3704ba33f0effd0b2ef349d2c7172d232223e5c2a6bf4c839488e2e586f98d10 +size 2177966216 diff --git a/model-00066-of-00073.safetensors b/model-00066-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..489bb7df2d3f84edf3ed41c96f8913fcf7013b4f --- /dev/null +++ b/model-00066-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:773902caf19552e9e83f10f2b6849a0c9f76d9ad342dc13d9c7997f2c06098de +size 4248207608 diff --git a/model-00067-of-00073.safetensors b/model-00067-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1e297ac8aa674c7fb58f6dde4b54178abf9e2b16 --- /dev/null +++ b/model-00067-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b87aba73ed161e6b22e104eb5fc20f7aa0217a49ef6a7d2b55ea2ab7df15ef75 +size 2177966216 diff --git a/model-00068-of-00073.safetensors b/model-00068-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6d7c64757983ad698b70eb7889c2834e928eb386 --- /dev/null +++ b/model-00068-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc99242a237167b469570ab2a85604e0a9a27674182c090fa8263441af535f67 +size 4248207608 diff --git a/model-00069-of-00073.safetensors b/model-00069-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0be87b214cca9ea8d227b96923f592e7503e8b1a --- /dev/null +++ b/model-00069-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:655fee25e250e3ce92e9b8d9b7514c6b851ce95fd43b6ce363178b69f377e683 +size 2177966216 diff --git a/model-00070-of-00073.safetensors b/model-00070-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..95e0703871be04a2f9d3717ae0872f1800a03336 --- /dev/null +++ b/model-00070-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e36762ad69a4e8cb5e6030c2ca03bd8a58724b3a083fc7010b765dc0b84c524 +size 4248207608 diff --git a/model-00071-of-00073.safetensors b/model-00071-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..af144a03205bd07557dec9af7759fe288244e576 --- /dev/null +++ b/model-00071-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78b756c3541582975ceaa34be42f5c29f71ada98c800fad32d97079d1126c794 +size 2177966216 diff --git a/model-00072-of-00073.safetensors b/model-00072-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5935f537dc27fb05c15fa5dd8ae64c6f13b8eb72 --- /dev/null +++ b/model-00072-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ad917bc799e71797e5d0dac2859b25398a1adf820df301651c2762f68856324 +size 4248207608 diff --git a/model-00073-of-00073.safetensors b/model-00073-of-00073.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d6e572c05607b3b203bc5249e19aac2506b808d5 --- /dev/null +++ b/model-00073-of-00073.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fd663e263ea6a4f2fcc281dae75eff6b2dafdba62a8d023de5d7457cd7e7000 +size 3282405744 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..6274cc1bd159aa75de771315558e5cac7dd8bea0 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|startoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|return|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..6ec3ef1795cbbda6b7cb7d1f114919cbe3fdd647 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0614fe83cadab421296e664e1f48f4261fa8fef6e03e63bb75c20f38e37d07d3 +size 27868174 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c021cddb0a9dd35b1bf83a9f145be2d9b3757891 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,183 @@ +{ + "added_tokens_decoder": { + "199998": { + "content": "<|startoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "199999": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200000": { + "content": "<|reserved_200000|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200001": { + "content": "<|reserved_200001|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200002": { + "content": "<|return|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200003": { + "content": "<|constrain|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200004": { + "content": "<|reserved_200004|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200005": { + "content": "<|channel|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200006": { + "content": "<|start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200007": { + "content": "<|end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200008": { + "content": "<|message|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200009": { + "content": "<|reserved_200009|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200010": { + "content": "<|reserved_200010|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200011": { + "content": "<|reserved_200011|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200012": { + "content": "<|call|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200013": { + "content": "<|reserved_200013|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200014": { + "content": "<|reserved_200014|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200015": { + "content": "<|reserved_200015|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200016": { + "content": "<|reserved_200016|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200017": { + "content": "<|reserved_200017|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200018": { + "content": "<|endofprompt|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|startoftext|>", + "clean_up_tokenization_spaces": false, + "eos_token": "<|return|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 1000000000000000019884624838656, + "pad_token": "<|endoftext|>", + "tokenizer_class": "PreTrainedTokenizerFast" +}