Python incompatibility error for VILA-LLAVA

[sandhiya-5cnetwork]

1. When I try to import the necessary libraries after cloning the NVlabs VILA reop

import logging
import os
import tempfile
import torch
from dotenv import load_dotenv
from llava.model.builder import load_pretrained_model
from llava.utils import disable_torch_init
from llava.mm_utils import process_images, tokenizer_image_token, KeywordsStoppingCriteria
from llava.conversation import conv_templates, SeparatorStyle
from llava.constants import IMAGE_TOKEN_INDEX
from experts.utils import load_image, get_modality

I got the following error:

---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
Cell In[6], line 6
      4 import torch
      5 from dotenv import load_dotenv
----> 6 from llava.model.builder import load_pretrained_model
      7 from llava.utils import disable_torch_init
      8 from llava.mm_utils import process_images, tokenizer_image_token, KeywordsStoppingCriteria

File ~/Sandhiya/VLM/MONAI/NVILA/Nllava/VILA/llava/model/__init__.py:1
----> 1 from .language_model.llava_llama import LlavaLlamaConfig, LlavaLlamaModel
      2 from .language_model.llava_topdown_llama import LlavaTopDownLlamaConfig, LlavaTopDownLlamaModel
      4 # FP8 related comments, development in progress (PI: ligeng zhu, haochen xi)
      5 # NOTE: VLM + LLM
      6 # from .language_model.qllava_qllama import QLlavaLlamaConfig, QLlavaLlamaModel
   (...)
      9 # NOTE: Linear + Activation -> fp8, haochen's iclr version
     10 # from .language_model.qmemllama import QMemLlamaConfig, QMemLlamaForCausalLM, QMemLlamaModel

File ~/Sandhiya/VLM/MONAI/NVILA/Nllava/VILA/llava/model/language_model/llava_llama.py:28
     26 from llava.model.loss import soft_cross_entropy
     27 from llava.model.utils.packing import set_seqlens_in_batch
---> 28 from llava.train.sequence_parallel.globals import get_pg_manager
     29 from llava.utils.logging import logger
     31 from ...train.utils import calculate_loss_weight

File ~/Sandhiya/VLM/MONAI/NVILA/Nllava/VILA/llava/train/sequence_parallel/__init__.py:1
----> 1 from .globals import get_pg_manager, set_pg_manager
      2 from .input_utils import extract_local_from_list, extract_local_input_ids, extract_local_position_ids

File ~/Sandhiya/VLM/MONAI/NVILA/Nllava/VILA/llava/train/sequence_parallel/globals.py:22
      1 # Copyright 2024 NVIDIA CORPORATION & AFFILIATES
      2 #
      3 # Licensed under the Apache License, Version 2.0 (the "License");
   (...)
     17 # This file is modified from https://github.com/feifeibear/long-context-attention
     18 # Implementation refers to USP Paper: https://arxiv.org/abs/2405.07719
     20 import os
---> 22 import deepspeed.comm as dist
     23 import torch
     26 class Singleton:

ModuleNotFoundError: No module named 'deepspeed'

2. Hence I installed deepspeed , using

pip install deepspeed

Which lead to the confilict:

ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
vila 2.0.0 requires pydantic<2,>=1, but you have pydantic 2.11.5 which is incompatible.

3. I commented the line - from llava.model.builder import load_pretrained_model - to check whether other packages are getting imported correct. Then I encountered the following error:

ImportError: cannot import name 'IMAGE_TOKEN_INDEX' from 'llava.constants' (/Sandhiya/VLM/MONAI/NVILA/Nllava/VILA/llava/constants.py)

This token constant is present in the original llava package

So, which llava should I use? The original llava or NVlabs VILA llava?