Upload 11 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+model.onnx_data filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

3rd_party_licenses.txt

@@ -0,0 +1,48 @@
+1: Name: TensorRT Model Optimizer - Windows
+Version: 0.19
+LicenseText: MIT License
+
+Copyright (c) 2023 MIT HAN Lab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+2: Name: onnxruntime-genai-directml
+Version: 0.4.0
+LicenseText:     MIT License
+
+    Copyright (c) Microsoft Corporation.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

License.txt

@@ -0,0 +1 @@
+GOVERNING TERMS: Use of this model is governed by the NVIDIA Open Model License Agreement (found at https://developer.download.nvidia.com/licenses/nvidia-open-model-license-agreement-june-2024.pdf). ADDITIONAL INFORMATION: Apache License, Version 2.0 (found at https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/apache-2.0.md).

Readme.txt

@@ -0,0 +1,10 @@
+To run inference with this model, please follow below steps -
+
+1. Install Nvidia Graphics driver R565 or higher. 
+2. Install python 3.10.11.
+3. Setup GenAI DirectML via
+    a. Install numpy - pip install numpy==2.1.0
+    b. Install GenAI - pip install onnxruntime-genai-directml==0.5.0
+4. Download inference script - curl https://github.com/microsoft/onnxruntime-genai/blob/rel-0.5.0/examples/python/phi3-qa.py -o phi3-qa.py
+5. Run inference - python phi3-qa.py -m <model-downloaded-path>
+    a. Enter prompt - "What is GenAI?"

genai_config.json

@@ -0,0 +1,54 @@
+{
+    "model": {
+        "bos_token_id": 2,
+        "context_length": 4096,
+        "decoder": {
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "provider_options": [
+                    {
+                        "dml": {}
+                    }
+                ]
+            },
+            "filename": "model.onnx",
+            "head_size": 128,
+            "hidden_size": 3072,
+            "inputs": {
+                "input_ids": "input_ids",
+                "attention_mask": "attention_mask",
+                "position_ids": "position_ids",
+                "past_key_names": "past_key_values.%d.key",
+                "past_value_names": "past_key_values.%d.value"
+            },
+            "outputs": {
+                "logits": "logits",
+                "present_key_names": "present.%d.key",
+                "present_value_names": "present.%d.value"
+            },
+            "num_attention_heads": 24,
+            "num_hidden_layers": 32,
+            "num_key_value_heads": 8
+        },
+        "eos_token_id": 3,
+        "pad_token_id": 3,
+        "type": "nemotron",
+        "vocab_size": 256000
+    },
+    "search": {
+        "diversity_penalty": 0.0,
+        "do_sample": false,
+        "early_stopping": true,
+        "length_penalty": 1.0,
+        "max_length": 4096,
+        "min_length": 0,
+        "no_repeat_ngram_size": 0,
+        "num_beams": 1,
+        "num_return_sequences": 1,
+        "past_present_share_buffer": true,
+        "repetition_penalty": 1.0,
+        "temperature": 1.0,
+        "top_k": 1,
+        "top_p": 1.0
+    }
+}

model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e5edb075cb4642ce920845ac7372861217418448ef3604a2c643ef7fa9bc0c2
+size 394332

model.onnx_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27dfc997144ec05ae7d374d11ab626b55d0a7184c933d7961065ae6a6650911e
+size 4498141184

quantization_log.txt

@@ -0,0 +1,45 @@
+(modelopt) PS E:\ModelOpt_Windows_Scripts_2\modelopt-windows-scripts\ONNX_PTQ> python quantize_script.py --model_name=nvidia/Nemotron-Mini-4B-Instruct  --onnx_path=E:\model_store\genai\nemotron-mini-4b-instruct-fp16-dml-genai\opset_21\model.onnx --output_path="E:\model_store\genai\nemotron-mini-4b-instruct-fp16-dml-genai\opset_21\default_quant_dml_ep_calib\model.onnx"
+
+--Quantize-Script-- algo=awq_lite, dataset=cnn, calib_size=32, batch_size=1, block_size=128, add-position-ids=True, past-kv=True, rcalib=False, device=cpu, use_zero_point=False
+
+
+
+--Quantize-Script-- awqlite_alpha_step=0.1, awqlite_fuse_nodes=False, awqlite_run_per_subgraph=False, awqclip_alpha_step=0.05, awqclip_alpha_min=0.5, awqclip_bsz_col=1024, calibration_eps=['dml']
+
+C:\Users\vrl\miniconda3\envs\modelopt\Lib\site-packages\transformers\models\auto\configuration_auto.py:1002: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.
+  warnings.warn(
+C:\Users\vrl\miniconda3\envs\modelopt\Lib\site-packages\transformers\models\auto\tokenization_auto.py:809: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.
+  warnings.warn(
+
+--Quantize-Script-- number_of_batched_samples=32, batch-input-ids-list-len=32, batched_attention_mask=32
+
+
+--Quantize-Script-- number of batched inputs = 32
+
+INFO:root:
+Quantizing the model....
+
+INFO:root:Quantization Mode: int4
+INFO:root:Finding quantizable weights and augmenting graph output with input activations
+INFO:root:Augmenting took 0.03900003433227539 seconds
+INFO:root:Saving the model took 35.37520098686218 seconds
+2024-11-05 06:08:38.8247274 [W:onnxruntime:, session_state.cc:1168 onnxruntime::VerifyEachNodeIsAssignedToAnEp] Some nodes were not assigned to the preferred execution providers which may or may not have an negative impact on performance. e.g. ORT explicitly assigns shape related ops to CPU to improve perf.
+2024-11-05 06:08:38.8385074 [W:onnxruntime:, session_state.cc:1170 onnxruntime::VerifyEachNodeIsAssignedToAnEp] Rerunning with verbose output on a non-minimal build will show node assignments.
+Getting activation names maps...: 100%|██████████████████████████████████████████████████████| 192/192 [00:00<?, ?it/s]
+Running AWQ scale search per node...: 100%|██████████████████████████████████████████| 192/192 [05:08<00:00,  1.61s/it]
+INFO:root:AWQ scale search took 308.7233784198761 seconds
+Quantizing the weights...: 100%|█████████████████████████████████████████████████████| 192/192 [00:05<00:00, 32.75it/s]
+INFO:root:Quantizing actual weights took 5.864110231399536 seconds
+INFO:root:Inserting DQ nodes and input_pre_quant_scale node using quantized weights and scales ...
+INFO:root:Inserting nodes took 0.1272134780883789 seconds
+INFO:root:Exporting the quantized graph ...
+Loading extension modelopt_round_and_pack_ext...
+
+INFO:root:Exporting took 33.892990589141846 seconds
+INFO:root:
+Quantization process took 394.4490396976471 seconds
+INFO:root:Saving to E:\model_store\genai\nemotron-mini-4b-instruct-fp16-dml-genai\opset_21\default_quant_dml_ep_calib\model.onnx took 33.43196678161621 seconds
+
+Done
+
+(modelopt) PS E:\ModelOpt_Windows_Scripts_2\modelopt-windows-scripts\ONNX_PTQ>

quantization_pip.txt

@@ -0,0 +1,307 @@
+(modelopt) PS E:\ModelOpt_Windows_Scripts_2\modelopt-windows-scripts\ONNX_PTQ> pip list
+Package                       Version                           Editable project location
+----------------------------- --------------------------------- -------------------------
+absl-py                       2.1.0
+accelerate                    0.34.2
+aiohttp                       3.9.5
+aiosignal                     1.3.1
+alabaster                     0.7.16
+annotated-types               0.6.0
+anyio                         4.3.0
+asttokens                     2.4.1
+astunparse                    1.6.3
+attrs                         23.2.0
+autodoc_pydantic              2.2.0
+Babel                         2.15.0
+backports.tarfile             1.1.1
+bandit                        1.7.9
+beautifulsoup4                4.12.3
+black                         24.4.2
+bleach                        6.1.0
+boto3                         1.35.0
+botocore                      1.35.0
+build                         1.2.1
+cachetools                    5.5.0
+certifi                       2024.2.2
+cffi                          1.17.0
+cfgv                          3.4.0
+chardet                       5.2.0
+charset-normalizer            3.3.2
+click                         8.1.7
+cloudpickle                   3.0.0
+colorama                      0.4.6
+colored                       2.2.4
+coloredlogs                   15.0.1
+comm                          0.2.2
+contourpy                     1.2.1
+coverage                      7.5.1
+cppimport                     22.8.2
+cuda-python                   12.3.0
+cupy-cuda12x                  13.3.0
+cycler                        0.12.1
+Cython                        3.0.10
+DataProperty                  1.0.1
+datasets                      2.19.0
+decorator                     5.1.1
+defusedxml                    0.7.1
+Deprecated                    1.2.14
+diffusers                     0.29.2
+dill                          0.3.8
+distlib                       0.3.8
+docutils                      0.20.1
+evaluate                      0.4.2
+executing                     2.0.1
+fastjsonschema                2.19.1
+fastrlock                     0.8.2
+filelock                      3.15.4
+fire                          0.6.0
+flatbuffers                   24.3.25
+fonttools                     4.51.0
+frozenlist                    1.4.1
+fsspec                        2024.3.1
+google-api-core               2.19.1
+google-api-python-client      2.141.0
+google-auth                   2.34.0
+google-auth-httplib2          0.2.0
+googleapis-common-protos      1.63.2
+GPUtil                        1.4.0
+h11                           0.14.0
+h5py                          3.11.0
+httplib2                      0.22.0
+huggingface-hub               0.24.0
+humanfriendly                 10.0
+identify                      2.5.36
+idna                          3.7
+imagesize                     1.4.1
+importlib_metadata            7.1.0
+iniconfig                     2.0.0
+intel-openmp                  2021.4.0
+ipython                       8.24.0
+ipywidgets                    8.1.2
+janus                         1.0.0
+jaraco.classes                3.4.0
+jaraco.context                5.3.0
+jaraco.functools              4.0.1
+jax                           0.4.31
+jaxlib                        0.4.31
+jedi                          0.19.1
+Jinja2                        3.1.3
+jmespath                      1.0.1
+joblib                        1.4.2
+jsonlines                     4.0.0
+jsonschema                    4.22.0
+jsonschema-specifications     2023.12.1
+jupyter_client                8.6.1
+jupyter_core                  5.7.2
+jupyterlab_pygments           0.3.0
+jupyterlab_widgets            3.0.10
+keyring                       25.2.0
+kiwisolver                    1.4.5
+lm_eval                       0.4.2
+lora                          0.3.0
+lxml                          5.2.2
+Mako                          1.3.5
+markdown-it-py                3.0.0
+MarkupSafe                    2.1.5
+maskprop                      0.1.3
+matplotlib                    3.8.4
+matplotlib-inline             0.1.7
+mbstrdecoder                  1.1.3
+mdurl                         0.1.2
+mistune                       3.0.2
+mkl                           2021.4.0
+ml-dtypes                     0.4.0
+more-itertools                10.2.0
+mpi4py                        3.1.6
+mpmath                        1.3.0
+multidict                     6.0.5
+multiprocess                  0.70.16
+mypy                          1.11.2
+mypy-extensions               1.0.0
+nbclient                      0.10.0
+nbconvert                     7.16.4
+nbformat                      5.10.4
+nbsphinx                      0.9.4
+networkx                      3.3
+neural_compressor             3.0.1
+nh3                           0.2.17
+ninja                         1.11.1.1
+nltk                          3.8.1
+nodeenv                       1.8.0
+numexpr                       2.10.0
+numpy                         1.26.4
+nvidia-cublas-cu12            12.5.2.13
+nvidia-cuda-nvrtc-cu12        12.5.40
+nvidia-cuda-runtime-cu12      12.4.127
+nvidia-cudnn-cu12             9.1.1.17
+nvidia-modelopt               0.12.2.dev106+g2ea947ec.d20240614 D:\__ammo_code__\modelopt
+onnx                          1.16.0
+onnx-graphsurgeon             0.5.2
+onnxconverter-common          1.14.0
+onnxmltools                   1.12.0
+onnxruntime-directml          1.20.0
+onnxscript                    0.1.0.dev20241104
+opencv-python                 4.9.0.80
+opencv-python-headless        4.10.0.84
+opt-einsum                    3.3.0
+optimum                       1.21.4
+outcome                       1.3.0.post0
+packaging                     24.1
+pandas                        2.2.2
+pandocfilters                 1.5.1
+parameterized                 0.9.0
+parso                         0.8.4
+pathspec                      0.12.1
+pathvalidate                  3.2.0
+pbr                           6.0.0
+peft                          0.13.0
+pillow                        10.2.0
+pip                           23.3.1
+pkginfo                       1.10.0
+platformdirs                  4.2.2
+pluggy                        1.5.0
+polygraphy                    0.49.9
+portalocker                   2.8.2
+pre-commit                    3.7.0
+prettytable                   3.11.0
+prompt-toolkit                3.0.43
+proto-plus                    1.24.0
+protobuf                      3.20.2
+psutil                        5.9.8
+PuLP                          2.8.0
+pure-eval                     0.2.2
+py-cpuinfo                    9.0.0
+pyarrow                       16.0.0
+pyarrow-hotfix                0.6
+pyasn1                        0.6.0
+pyasn1_modules                0.4.0
+pybind11                      2.12.0
+pycocotools                   2.0.8
+pycparser                     2.22
+pycuda                        2024.1.2
+pydantic                      2.7.1
+pydantic_core                 2.18.2
+pydantic-settings             2.2.1
+Pygments                      2.17.2
+pynvml                        11.5.0
+pypandoc                      1.13
+pyparsing                     3.1.2
+pyproject-api                 1.7.1
+pyproject_hooks               1.1.0
+pyreadline3                   3.4.1
+PySocks                       1.7.1
+pytablewriter                 1.2.0
+pytest                        8.2.0
+pytest-asyncio                0.23.6
+pytest-cov                    5.0.0
+pytest-timeout                2.3.1
+python-dateutil               2.9.0.post0
+python-dotenv                 1.0.1
+python-magic                  0.4.27
+pytools                       2024.1.13
+pytz                          2024.1
+pywin32                       306
+pywin32-ctypes                0.2.2
+PyYAML                        6.0.1
+pyzmq                         26.0.3
+readme_renderer               43.0
+referencing                   0.35.1
+regex                         2024.4.28
+requests                      2.31.0
+requests-toolbelt             1.0.0
+rfc3986                       2.0.0
+rich                          13.7.1
+rouge-score                   0.1.2
+rpds-py                       0.18.1
+rsa                           4.9
+ruff                          0.6.4
+s3transfer                    0.10.2
+sacrebleu                     2.4.2
+safetensors                   0.4.3
+schema                        0.7.7
+scikit-cuda                   0.5.3
+scikit-learn                  1.5.0
+scipy                         1.13.0
+seaborn                       0.13.2
+selenium                      4.23.1
+sentencepiece                 0.2.0
+setuptools                    68.2.2
+setuptools-scm                8.1.0
+six                           1.16.0
+skipy                         0.2.0
+skl2onnx                      1.17.0
+sniffio                       1.3.1
+snowballstemmer               2.2.0
+sortedcontainers              2.4.0
+soupsieve                     2.5
+Sphinx                        7.2.6
+sphinx-autobuild              2024.4.16
+sphinx-copybutton             0.5.2
+sphinx_inline_tabs            2023.4.21
+sphinx-rtd-theme              2.0.0
+sphinx-togglebutton           0.3.2
+sphinxcontrib-applehelp       1.0.8
+sphinxcontrib-devhelp         1.0.6
+sphinxcontrib-htmlhelp        2.0.5
+sphinxcontrib-jquery          4.1
+sphinxcontrib-jsmath          1.0.1
+sphinxcontrib-qthelp          1.0.7
+sphinxcontrib-serializinghtml 1.1.10
+sqlitedict                    2.1.0
+stack-data                    0.6.3
+starlette                     0.37.2
+stevedore                     5.2.0
+StrEnum                       0.4.15
+sympy                         1.12
+tabledata                     1.3.3
+tabulate                      0.9.0
+tbb                           2021.12.0
+tcolorpy                      0.1.6
+tensorrt                      10.1.0
+tensorrt-bindings             9.3.0.post12.dev1
+tensorrt-cu12                 10.1.0
+tensorrt-cu12_bindings        10.1.0
+tensorrt-cu12_libs            10.1.0
+termcolor                     2.4.0
+threadpoolctl                 3.5.0
+timm                          0.9.16
+tinycss2                      1.3.0
+tokenizers                    0.20.2
+toml                          0.10.2
+torch                         2.3.0+cu118
+torchaudio                    2.3.0+cu118
+torchprofile                  0.0.4
+torchsurgeon                  0.1.2
+torchvision                   0.18.0+cu118
+tornado                       6.4
+tox                           4.17.1
+tox-current-env               0.0.12
+tqdm                          4.66.2
+tqdm-multiprocess             0.0.11
+traitlets                     5.14.3
+transformers                  4.46.1
+trio                          0.26.2
+trio-websocket                0.11.1
+twine                         5.0.0
+typepy                        1.3.2
+typing_extensions             4.11.0
+tzdata                        2024.1
+uritemplate                   4.1.1
+urllib3                       2.2.1
+uvicorn                       0.29.0
+virtualenv                    20.26.3
+watchfiles                    0.21.0
+wcwidth                       0.2.13
+webencodings                  0.5.1
+websocket-client              1.8.0
+websockets                    12.0
+wheel                         0.41.2
+widgetsnbextension            4.0.10
+word2number                   1.1
+wrapt                         1.16.0
+wsproto                       1.2.0
+xxhash                        3.4.1
+yarl                          1.9.4
+zipp                          3.18.1
+zstandard                     0.22.0
+(modelopt) PS E:\ModelOpt_Windows_Scripts_2\modelopt-windows-scripts\ONNX_PTQ>

special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91fd32f29cfeb8fee9535681f551697b066992c9d950d810651801be24f39e93
+size 34809710