Traceback (most recent call last):
  File "/llm-perf-backend/llm_perf/common/benchmark_runner.py", line 297, in execute_and_log_benchmark
    benchmark_report = Benchmark.launch(benchmark_config)
  File "/usr/local/lib/python3.10/dist-packages/optimum_benchmark/benchmark/base.py", line 51, in launch
    report = launcher.launch(worker=Benchmark.run, worker_args=[config])
  File "/usr/local/lib/python3.10/dist-packages/optimum_benchmark/launchers/process/launcher.py", line 66, in launch
    raise ChildProcessError(response["traceback"])
ChildProcessError: Traceback (most recent call last):
  File "/usr/local/lib/python3.10/dist-packages/hydra/_internal/utils.py", line 644, in _locate
    obj = getattr(obj, part)
AttributeError: module 'optimum_benchmark.backends.pytorch' has no attribute 'backend'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/dist-packages/transformers/utils/import_utils.py", line 1603, in _get_module
    return importlib.import_module("." + module_name, self.__name__)
  File "/usr/lib/python3.10/importlib/__init__.py", line 126, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
  File "/usr/local/lib/python3.10/dist-packages/transformers/trainer.py", line 215, in <module>
    from peft import PeftModel
  File "/usr/local/lib/python3.10/dist-packages/peft/__init__.py", line 22, in <module>
    from .auto import (
  File "/usr/local/lib/python3.10/dist-packages/peft/auto.py", line 32, in <module>
    from .mapping import MODEL_TYPE_TO_PEFT_MODEL_MAPPING
  File "/usr/local/lib/python3.10/dist-packages/peft/mapping.py", line 22, in <module>
    from peft.tuners.xlora.model import XLoraModel
  File "/usr/local/lib/python3.10/dist-packages/peft/tuners/__init__.py", line 21, in <module>
    from .lora import LoraConfig, LoraModel, LoftQConfig, LoraRuntimeConfig
  File "/usr/local/lib/python3.10/dist-packages/peft/tuners/lora/__init__.py", line 20, in <module>
    from .model import LoraModel
  File "/usr/local/lib/python3.10/dist-packages/peft/tuners/lora/model.py", line 50, in <module>
    from .awq import dispatch_awq
  File "/usr/local/lib/python3.10/dist-packages/peft/tuners/lora/awq.py", line 26, in <module>
    from awq.modules.linear import WQLinear_GEMM
  File "/usr/local/lib/python3.10/dist-packages/awq/__init__.py", line 2, in <module>
    from awq.models.auto import AutoAWQForCausalLM
  File "/usr/local/lib/python3.10/dist-packages/awq/models/__init__.py", line 1, in <module>
    from .mpt import MptAWQForCausalLM
  File "/usr/local/lib/python3.10/dist-packages/awq/models/mpt.py", line 1, in <module>
    from .base import BaseAWQForCausalLM
  File "/usr/local/lib/python3.10/dist-packages/awq/models/base.py", line 52, in <module>
    from awq.quantize.quantizer import AwqQuantizer
  File "/usr/local/lib/python3.10/dist-packages/awq/quantize/quantizer.py", line 10, in <module>
    from awq.quantize.scale import apply_scale, apply_clip
  File "/usr/local/lib/python3.10/dist-packages/awq/quantize/scale.py", line 8, in <module>
    from transformers.models.llama.modeling_llama import LlamaRMSNorm
  File "/usr/local/lib/python3.10/dist-packages/transformers/models/llama/modeling_llama.py", line 32, in <module>
    from ...modeling_flash_attention_utils import _flash_attention_forward
  File "/usr/local/lib/python3.10/dist-packages/transformers/modeling_flash_attention_utils.py", line 27, in <module>
    from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input  # noqa
  File "/usr/local/lib/python3.10/dist-packages/flash_attn/__init__.py", line 3, in <module>
    from flash_attn.flash_attn_interface import (
  File "/usr/local/lib/python3.10/dist-packages/flash_attn/flash_attn_interface.py", line 15, in <module>
    import flash_attn_2_cuda as flash_attn_gpu
ImportError: /usr/local/lib/python3.10/dist-packages/flash_attn_2_cuda.cpython-310-x86_64-linux-gnu.so: undefined symbol: _ZNK3c1011StorageImpl27throw_data_ptr_access_errorEv

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/dist-packages/hydra/_internal/utils.py", line 650, in _locate
    obj = import_module(mod)
  File "/usr/lib/python3.10/importlib/__init__.py", line 126, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
  File "/usr/local/lib/python3.10/dist-packages/optimum_benchmark/backends/pytorch/backend.py", line 10, in <module>
    from transformers import (
  File "<frozen importlib._bootstrap>", line 1075, in _handle_fromlist
  File "/usr/local/lib/python3.10/dist-packages/transformers/utils/import_utils.py", line 1593, in __getattr__
    module = self._get_module(self._class_to_module[name])
  File "/usr/local/lib/python3.10/dist-packages/transformers/utils/import_utils.py", line 1605, in _get_module
    raise RuntimeError(
RuntimeError: Failed to import transformers.trainer because of the following error (look up to see its traceback):
/usr/local/lib/python3.10/dist-packages/flash_attn_2_cuda.cpython-310-x86_64-linux-gnu.so: undefined symbol: _ZNK3c1011StorageImpl27throw_data_ptr_access_errorEv

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/dist-packages/optimum_benchmark/launchers/process/launcher.py", line 103, in target
    report = worker(*worker_args)
  File "/usr/local/lib/python3.10/dist-packages/optimum_benchmark/benchmark/base.py", line 69, in run
    backend_factory: Type[Backend] = get_class(backend_config._target_)
  File "/usr/local/lib/python3.10/dist-packages/hydra/utils.py", line 40, in get_class
    raise e
  File "/usr/local/lib/python3.10/dist-packages/hydra/utils.py", line 31, in get_class
    cls = _locate(path)
  File "/usr/local/lib/python3.10/dist-packages/hydra/_internal/utils.py", line 658, in _locate
    raise ImportError(
ImportError: Error loading 'optimum_benchmark.backends.pytorch.backend.PyTorchBackend':
RuntimeError('Failed to import transformers.trainer because of the following error (look up to see its traceback):\n/usr/local/lib/python3.10/dist-packages/flash_attn_2_cuda.cpython-310-x86_64-linux-gnu.so: undefined symbol: _ZNK3c1011StorageImpl27throw_data_ptr_access_errorEv')

