fix(infra): remove transformers dependency for api server (#5441)

Co-authored-by: Edwin Luo <edwinluo@3ef5a334-3d74-4dbf-b1c8-d57dc87d5638.attlocal.net>
Co-authored-by: Claude <noreply@anthropic.com>

Author: 3 people

backend/onyx/natural_language_processing/utils.py

@@ -5,7 +5,6 @@
 
 from tokenizers import Encoding  # type: ignore
 from tokenizers import Tokenizer  # type: ignore
-from transformers import logging as transformer_logging  # type:ignore
 
 from onyx.configs.model_configs import DOC_EMBEDDING_CONTEXT_SIZE
 from onyx.configs.model_configs import DOCUMENT_ENCODER_MODEL
@@ -16,10 +15,8 @@
 TRIM_SEP_PAT = "\n... {n} tokens removed...\n"
 
 logger = setup_logger()
-transformer_logging.set_verbosity_error()
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"
-os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"
 
 
 class BaseTokenizer(ABC):

backend/requirements/default.txt

@@ -83,7 +83,6 @@ supervisor==4.2.5
 RapidFuzz==3.13.0
 tiktoken==0.7.0
 timeago==1.0.16
-transformers==4.49.0
 unstructured==0.15.1
 unstructured-client==0.25.4
 uvicorn==0.35.0