Huggingface space getting error after a token revocation
Hello we are new to huggingface and started to get errors on a working space after we received a notification that HF tokens were compromised.  I have posted as much detail as I can with regards to our settings and build logs.Not sure where we can go from here as we are pretty stuck on this error.We have an organization with private spaces, datasets, and models running on a NVidia A10Gsmall. We use a HF_Token as a secret and we are running a ai space called camille.title: Camille Traineremoji:colorFrom: graycolorTo: graysdk: gradiosdk_version: 4.36.1app_file: app.pypinned: truelicense: mitthis is from our app.pyimport gradio as grfrom docs import title_and_step1from camille_helpers import fetch_upstream_dataset, fetch_upstream_user_hashes, reindexfrom REDACTED_TOKEN import deploy_model, fetch_REDACTED_TOKEN, update_REDACTED_TOKEN training_helpers import train_model…Camille-trainerimport osimport timeimport requestsimport gradio as grfrom docs import step2from REDACTED_TOKEN import get_inference_endpoint, wake_serviceBASE_URL = os.environ.get(…REDACTED_TOKEN.pyimport timeimport gradio as grfrom typing import List, Dict, Anyfrom dataclasses import asdictfrom datasets import Dataset, load_dataset, DownloadModefrom huggingface_hub import (InferenceEndpoint,list_repo_refs,list_inference_endpoints,create_inference_endpoint,)from huggingface_hub.utils._errors import HfHubHTTPError, RepositoryNotFoundErrorfrom docs import step3, step5download_mode = DownloadMode.FORCE_REDOWNLOAD…Local-requrements.txtdatasetshuggingface-hub>=0.23.3protobuf<4click<8.1pydantic~=1.0gradio[oauth]==4.36.1uvicorn>=0.14.0spaces==0.17.0model.pyfrom sentence_transformers import SentenceTransformer, modelsimport torchdef get_base_model():device = torch.device(“cuda” if torch.cuda.is_available() else “cpu”)word_embedding_model = models.Transformer("BAAI/bge-base-en-v1.5", max_seq_length=512)

word_embedding_model.tokenizer.add_tokens(['[TURN]'], special_tokens=True)
word_embedding_model.tokenizer.truncation_side = 'left'
word_embedding_model.auto_model.resize_token_embeddings(len(word_embedding_model.tokenizer))

pooling_model = models.Pooling(
    word_embedding_model.get_word_embedding_dimension(), pooling_mode="cls"
)

model = SentenceTransformer(modules=[word_embedding_model, pooling_model], device=device)

return model…requirements.txt–extra-index-urlhttps://download.pytorch.org/whl/cu113torchsentence-transformershuggingface-hub==0.23.3training_helper.pyimport jsonimport gradio as grimport pandas as pdfrom sentence_transformers import InputExamplefrom sentence_transformers.losses import OnlineContrastiveLossfrom sentence_transformers.evaluation import EmbeddingSimilarityEvaluatorfrom datasets import load_dataset, DownloadModefrom torch.utils.data import DataLoaderfrom huggingface_hub import create_repo, upload_folder, REDACTED_TOKEN_downloadfrom docs import step4from model import get_base_modelfrom REDACTED_TOKEN import get_repo_main_branchMODEL_PATH = “./model”BATCH_SIZE = 32We believe there is a problem with torch . When we get to the train model section we get this error.raise ImportError(ImportError: Using theTrainerwithPyTorchrequiresaccelerate>=0.20.1: Please runpip install transformers[torch]orpip install accelerate -UWe’ve ran these command with no successThis is the full error===== Application Startup at 2024-06-17 18:39:58 =====/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead._torch_pytree._register_pytree_node(/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead._torch_pytree._register_pytree_node(Running on local URL:http://0.0.0.0:7860To create a public link, setshare=Trueinlaunch()./home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/datasets/dataset_dict.py:1682: FutureWarning: ‘branch’ was deprecated in favor of ‘revision’ in version 2.15.0 and will be removed in 3.0.0.You can remove this warning by passing ‘revision=main’ instead.warnings.warn(/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning:resume_downloadis deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, useforce_download=True.warnings.warn(/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead._torch_pytree._register_pytree_node(Traceback (most recent call last):File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/gradio/queueing.py”, line 532, in process_eventsresponse = await route_utils.call_process_api(File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/gradio/route_utils.py”, line 276, in call_process_apioutput = await app.get_blocks().process_api(File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/gradio/blocks.py”, line 1928, in process_apiresult = await self.call_function(File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/gradio/blocks.py”, line 1514, in call_functionprediction = await anyio.to_thread.run_sync(File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/anyio/to_thread.py”, line 56, in run_syncreturn await get_async_backend().run_sync_in_worker_thread(File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/anyio/_backends/_asyncio.py”, line 2177, in run_sync_in_worker_threadreturn await futureFile “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/anyio/_backends/_asyncio.py”, line 859, in runresult = context.run(func, *args)File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/gradio/utils.py”, line 832, in wrapperresponse = f(*args, **kwargs)File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/gradio/utils.py”, line 832, in wrapperresponse = f(*args, **kwargs)File “/home/user/app/training_helpers.py”, line 96, in train_modelmodel.fit(File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/sentence_transformers/fit_mixin.py”, line 304, in fitargs = SentenceTransformerTrainingArguments(File “”, line 119, ininitFile “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/sentence_transformers/training_args.py”, line 73, inpost_initsuper().post_init()File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/transformers/training_args.py”, line 1442, inpost_initand (self.device.type != “cuda”)File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/transformers/training_args.py”, line 1887, in devicereturn self._setup_devicesFile “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/transformers/utils/generic.py”, line 54, ingetcached = self.fget(obj)File “/home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages/transformers/training_args.py”, line 1787, in _setup_devicesraise ImportError(ImportError: Using theTrainerwithPyTorchrequiresaccelerate>=0.20.1: Please runpip install transformers[torch]orpip install accelerate -UHere is the build log===== Build Queued at 2024-06-12 19:55:45 / Commit SHA: 6c2158a =====→ FROMdocker.io/nvidia/cuda:12.3.2-cudnn9-devel-ubuntu22.04@sha256:fb1ad20f2552f5b3aafb2c9c478ed57da95e2bb027d15218d7a55b3a0e4b4413DONE 0.0s→ COPY --chown=1000:1000 --from=root / /CACHED→ RUN apt-get update && apt-get install -y fakeroot &&     mv /usr/bin/apt-get /usr/bin/.apt-get &&     echo ‘#!/usr/bin/env sh\nfakeroot /usr/bin/.apt-get $@’ > /usr/bin/apt-get &&     chmod +x /usr/bin/apt-get && 	rm -rf /var/lib/apt/lists/* && 	useradd -m -u 1000 userCACHED→ RUN pip install --no-cache-dir pip==22.3.1 && 	pip install --no-cache-dir 	datasets 	“huggingface-hub>=0.19” “hf-transfer>=0.1.4” “protobuf<4” “click<8.1” “pydantic~=1.0”CACHED→ RUN curlhttps://pyenv.run| bashCACHED→ WORKDIR /home/user/appCACHED→ RUN apt-get update && apt-get install -y 	git rsync 	make build-essential libssl-dev zlib1g-dev 	libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm 	libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev git-lfs  	ffmpeg libsm6 libxext6 cmake libgl1-mesa-glx 	&& rm -rf /var/lib/apt/lists/* 	&& git lfs installCACHED→ RUN pyenv install 3.10 && 	pyenv global 3.10 && 	pyenv rehashCACHED→ RUN --mount=target=/tmp/requirements.txt,source=requirements.txt     pip install --no-cache-dir -r /tmp/requirements.txtCACHED→ Restoring cacheDONE 46.0s→ RUN pip install --no-cache-dir 	gradio[oauth]==4.36.1 	“uvicorn>=0.14.0” 	spacesCollecting gradio[oauth]==4.36.1Downloading gradio-4.36.1-py3-none-any.whl (12.3 MB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 12.3/12.3 MB 193.1 MB/s eta 0:00:00Collecting uvicorn>=0.14.0Downloading uvicorn-0.30.1-py3-none-any.whl (62 kB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.4/62.4 kB 308.7 MB/s eta 0:00:00Collecting spacesDownloading spaces-0.28.3-py3-none-any.whl (18 kB)Requirement already satisfied: numpy<3.0,>=1.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (1.26.4)Collecting importlib-resources<7.0,>=1.3Downloading importlib_resources-6.4.0-py3-none-any.whl (38 kB)Requirement already satisfied: markupsafe~=2.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (2.1.5)Requirement already satisfied: urllib3~=2.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (2.2.1)Requirement already satisfied: pandas<3.0,>=1.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (2.2.2)Requirement already satisfied: packaging in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (24.0)Collecting altair<6.0,>=4.2.0Downloading altair-5.3.0-py3-none-any.whl (857 kB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 857.8/857.8 kB 400.3 MB/s eta 0:00:00Collecting huggingface-hub>=0.19.3Downloading huggingface_hub-0.23.3-py3-none-any.whl (401 kB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 401.7/401.7 kB 431.4 MB/s eta 0:00:00Requirement already satisfied: pyyaml<7.0,>=5.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (6.0.1)Collecting fastapiDownloading fastapi-0.111.0-py3-none-any.whl (91 kB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 92.0/92.0 kB 337.9 MB/s eta 0:00:00Requirement already satisfied: jinja2<4.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (3.1.4)Collecting httpx>=0.24.1Downloading httpx-0.27.0-py3-none-any.whl (75 kB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 75.6/75.6 kB 353.1 MB/s eta 0:00:00Collecting aiofiles<24.0,>=22.0Downloading aiofiles-23.2.1-py3-none-any.whl (15 kB)Collecting ruff>=0.2.2Downloading ruff-0.4.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.8 MB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 8.8/8.8 MB 214.5 MB/s eta 0:00:00Collecting typer<1.0,>=0.12Downloading typer-0.12.3-py3-none-any.whl (47 kB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 47.2/47.2 kB 287.9 MB/s eta 0:00:00Collecting ffmpyDownloading ffmpy-0.3.2.tar.gz (5.5 kB)Preparing metadata (setup.py): startedPreparing metadata (setup.py): finished with status ‘done’Requirement already satisfied: typing-extensions~=4.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (4.12.1)Collecting gradio-client==1.0.1Downloading gradio_client-1.0.1-py3-none-any.whl (318 kB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 318.1/318.1 kB 396.0 MB/s eta 0:00:00Collecting matplotlib~=3.0Downloading matplotlib-3.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.3 MB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 8.3/8.3 MB 170.0 MB/s eta 0:00:00Collecting python-multipart>=0.0.9Downloading python_multipart-0.0.9-py3-none-any.whl (22 kB)Collecting tomlkit==0.12.0Downloading tomlkit-0.12.0-py3-none-any.whl (37 kB)Requirement already satisfied: pillow<11.0,>=8.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio[oauth]==4.36.1) (10.3.0)Collecting semantic-version~=2.0Downloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)Collecting pydubDownloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)Collecting orjson~=3.0Downloading orjson-3.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (142 kB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 142.7/142.7 kB 376.0 MB/s eta 0:00:00Collecting pydantic>=2.0Downloading pydantic-2.7.4-py3-none-any.whl (409 kB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 409.0/409.0 kB 428.4 MB/s eta 0:00:00Collecting authlibDownloading Authlib-1.3.1-py2.py3-none-any.whl (223 kB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 223.8/223.8 kB 419.3 MB/s eta 0:00:00Collecting itsdangerousDownloading itsdangerous-2.2.0-py3-none-any.whl (16 kB)Collecting websockets<12.0,>=10.0Downloading websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 129.9/129.9 kB 293.8 MB/s eta 0:00:00Requirement already satisfied: fsspec in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from gradio-client==1.0.1->gradio[oauth]==4.36.1) (2024.3.1)Requirement already satisfied: click>=7.0 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from uvicorn>=0.14.0) (8.0.4)Collecting h11>=0.8Downloading h11-0.14.0-py3-none-any.whl (58 kB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 58.3/58.3 kB 315.5 MB/s eta 0:00:00Collecting psutil<6,>=2Downloading psutil-5.9.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (288 kB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 288.2/288.2 kB 332.1 MB/s eta 0:00:00Requirement already satisfied: requests<3.0,>=2.19 in /home/user/.pyenv/versions/3.10.14/lib/python3.10/site-packages (from spaces) (2.32.3)Collecting toolzDownloading toolz-0.12.1-py3-none-any.whl (56 kB)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 56.1/56.1 kB 309.1 MB/s eta 0:00:00Collecting jsonschema>=3.0Downloading jsonschema-4.22.0-py3-none-any.whl (88 kB)