add environment_h100.yml

README.md

@@ -18,6 +18,16 @@ After installing conda, you need to set up a new conda environment for MedVersa
 conda env create -f environment.yml
 conda activate medversa
 ```
+The above `environment.yml` has been validated on NVIDIA A100 GPUs. If you have more advanced cards, e.g., NVIDIA H100 GPUs, you will need `environment_h100.yml`:
+``` shell
+conda env create -f environment_h100.yml
+conda activate medversa
+```
+
+If you encounter an issue of opencv, you may need to reinstall opencv-python:
+``` shell
+pip install opencv-contrib-python
+```
 
 ## Inference
 ``` python

environment.yml

@@ -475,5 +475,4 @@ dependencies:
       - xxhash==3.3.0
       - yarl==1.8.2
       - zipp==3.14.0
-prefix: /home/zhouhy/anaconda3/envs/medversa
-
+prefix: /home/zhouhy/anaconda3/envs/medversa

environment_h100.yml

@@ -0,0 +1,526 @@
+name: medversa
+channels:
+  - pytorch
+  - nvidia
+  - nvidia/label/cuda-11.8.0
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - abseil-cpp=20211102.0=h27087fc_1
+  - aiosignal=1.3.1=pyhd8ed1ab_0
+  - argcomplete=1.12.0=py_0
+  - arrow-cpp=11.0.0=py39h613000e_0
+  - asttokens=2.2.1=pyhd8ed1ab_0
+  - async-timeout=4.0.2=pyhd8ed1ab_0
+  - atk-1.0=2.36.0=ha1a6a79_0
+  - aws-c-common=0.4.57=he6710b0_1
+  - aws-c-event-stream=0.1.6=h2531618_5
+  - aws-checksums=0.1.9=he6710b0_0
+  - aws-sdk-cpp=1.8.185=hce553d0_0
+  - blas=1.0=mkl
+  - boost-cpp=1.70.0=ha2d47e9_1
+  - boto=2.49.0=py39h06a4308_0
+  - bottleneck=1.3.5=py39h7deecbd_0
+  - brotli-python=1.0.9=py39h6a678d5_7
+  - brotlipy=0.7.0=py39h27cfd23_1003
+  - bzip2=1.0.8=h7b6447c_0
+  - c-ares=1.18.1=h7f8727e_0
+  - ca-certificates=2024.3.11=h06a4308_0
+  - cairo=1.16.0=hb05425b_5
+  - certifi=2024.2.2=py39h06a4308_0
+  - cffi=1.15.1=py39h5eee18b_3
+  - colorama=0.4.6=pyhd8ed1ab_0
+  - crcmod=1.7=py39hb9d737c_1008
+  - cryptography=41.0.2=py39h774aba0_0
+  - cuda-cccl=11.8.89=0
+  - cuda-command-line-tools=11.8.0=0
+  - cuda-compiler=11.8.0=0
+  - cuda-cudart=11.8.89=0
+  - cuda-cudart-dev=11.8.89=0
+  - cuda-cuobjdump=11.8.86=0
+  - cuda-cupti=11.8.87=0
+  - cuda-cuxxfilt=11.8.86=0
+  - cuda-documentation=11.8.86=0
+  - cuda-driver-dev=11.8.89=0
+  - cuda-gdb=11.8.86=0
+  - cuda-libraries=11.8.0=0
+  - cuda-libraries-dev=11.8.0=0
+  - cuda-memcheck=11.8.86=0
+  - cuda-nsight=11.8.86=0
+  - cuda-nsight-compute=11.8.0=0
+  - cuda-nvcc=11.8.89=0
+  - cuda-nvdisasm=11.8.86=0
+  - cuda-nvml-dev=11.8.86=0
+  - cuda-nvprof=11.8.87=0
+  - cuda-nvprune=11.8.86=0
+  - cuda-nvrtc=11.8.89=0
+  - cuda-nvrtc-dev=11.8.89=0
+  - cuda-nvtx=11.8.86=0
+  - cuda-nvvp=11.8.87=0
+  - cuda-profiler-api=11.8.86=0
+  - cuda-runtime=11.8.0=0
+  - cuda-sanitizer-api=11.8.86=0
+  - cuda-toolkit=11.8.0=0
+  - cuda-tools=11.8.0=0
+  - cuda-visual-tools=11.8.0=0
+  - curl=7.87.0=h5eee18b_0
+  - dataclasses=0.8=pyhc8e2a94_3
+  - datasets=2.14.3=pyhd8ed1ab_0
+  - dill=0.3.7=pyhd8ed1ab_0
+  - executing=1.2.0=pyhd8ed1ab_0
+  - expat=2.4.9=h6a678d5_0
+  - fasteners=0.16.3=pyhd3eb1b0_0
+  - ffmpeg=4.3=hf484d3e_0
+  - filelock=3.9.0=py39h06a4308_0
+  - flit-core=3.6.0=pyhd3eb1b0_0
+  - font-ttf-dejavu-sans-mono=2.37=hd3eb1b0_0
+  - font-ttf-inconsolata=2.001=hcb22688_0
+  - font-ttf-source-code-pro=2.030=hd3eb1b0_0
+  - font-ttf-ubuntu=0.83=h8b1ccd4_0
+  - fontconfig=2.14.1=h52c9d5c_1
+  - fonts-anaconda=1=h8fa9717_0
+  - fonts-conda-ecosystem=1=hd3eb1b0_0
+  - freetype=2.12.1=h4a9f257_0
+  - fribidi=1.0.10=h7b6447c_0
+  - frozenlist=1.3.3=py39h5eee18b_0
+  - fsspec=2024.3.1=pyhca7485f_0
+  - gcs-oauth2-boto-plugin=3.0=pyhd8ed1ab_1
+  - gdbm=1.18=hd4cb3f1_4
+  - gdk-pixbuf=2.42.10=h5eee18b_0
+  - gds-tools=1.4.0.31=0
+  - gettext=0.21.0=hf68c758_0
+  - gflags=2.2.2=he1b5a44_1004
+  - giflib=5.2.1=h5eee18b_3
+  - git=2.34.1=pl5262hc120c5b_0
+  - git-lfs=3.5.1=ha770c72_0
+  - glib=2.69.1=he621ea3_2
+  - glog=0.5.0=h48cff8f_0
+  - gmp=6.2.1=h295c915_3
+  - gmpy2=2.1.2=py39heeb90bb_0
+  - gnutls=3.6.15=he1e5248_0
+  - gobject-introspection=1.72.0=py39hbb6d50b_2
+  - google-apitools=0.5.32=pyhd8ed1ab_0
+  - google-reauth=0.1.1=pyhd3deb0d_0
+  - graphite2=1.3.14=h295c915_1
+  - graphviz=2.50.0=h1b29801_1
+  - grpc-cpp=1.46.1=h33aed49_1
+  - gsutil=5.27=pyhd8ed1ab_0
+  - gtk2=2.24.33=h73c1081_2
+  - gts=0.7.6=hb67d8dd_3
+  - harfbuzz=4.3.0=hf52aaf7_1
+  - httplib2=0.20.4=pyhd8ed1ab_0
+  - huggingface_hub=0.20.3=py39h06a4308_0
+  - icu=58.2=he6710b0_3
+  - idna=3.4=py39h06a4308_0
+  - importlib_metadata=6.8.0=hd8ed1ab_0
+  - intel-openmp=2023.1.0=hdb19cb5_46305
+  - jinja2=3.1.2=py39h06a4308_0
+  - joblib=1.2.0=py39h06a4308_0
+  - jpeg=9e=h5eee18b_1
+  - krb5=1.19.4=h568e23c_0
+  - lame=3.100=h7b6447c_0
+  - lcms2=2.12=h3be6417_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - lerc=3.0=h295c915_0
+  - libbrotlicommon=1.0.9=h166bdaf_7
+  - libbrotlidec=1.0.9=h166bdaf_7
+  - libbrotlienc=1.0.9=h166bdaf_7
+  - libcublas=11.11.3.6=0
+  - libcublas-dev=11.11.3.6=0
+  - libcufft=10.9.0.58=0
+  - libcufft-dev=10.9.0.58=0
+  - libcufile=1.7.1.12=0
+  - libcufile-dev=1.4.0.31=0
+  - libcurand=10.3.3.129=0
+  - libcurand-dev=10.3.0.86=0
+  - libcurl=7.87.0=h91b91d3_0
+  - libcusolver=11.4.1.48=0
+  - libcusolver-dev=11.4.1.48=0
+  - libcusparse=11.7.5.86=0
+  - libcusparse-dev=11.7.5.86=0
+  - libdeflate=1.17=h5eee18b_0
+  - libedit=3.1.20221030=h5eee18b_0
+  - libev=4.33=h7f8727e_1
+  - libevent=2.1.10=h9b69904_4
+  - libffi=3.4.2=h6a678d5_6
+  - libgcc=7.2.0=h69d50b8_2
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgd=2.3.3=h6a678d5_3
+  - libgomp=11.2.0=h1234567_1
+  - libiconv=1.16=h7f8727e_2
+  - libidn2=2.3.4=h5eee18b_0
+  - libnghttp2=1.46.0=hce63b2e_0
+  - libnpp=11.8.0.86=0
+  - libnpp-dev=11.8.0.86=0
+  - libnvjpeg=11.9.0.86=0
+  - libnvjpeg-dev=11.9.0.86=0
+  - libpng=1.6.39=h5eee18b_0
+  - libprotobuf=3.20.3=he621ea3_0
+  - librsvg=2.54.4=h36cc946_2
+  - libssh2=1.10.0=h8f2d780_0
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - libtasn1=4.19.0=h5eee18b_0
+  - libthrift=0.15.0=he6d91bd_0
+  - libtiff=4.5.0=h6a678d5_2
+  - libtool=2.4.6=h6a678d5_1009
+  - libunistring=0.9.10=h27cfd23_0
+  - libuuid=1.41.5=h5eee18b_0
+  - libwebp=1.2.4=h11a3e52_1
+  - libwebp-base=1.2.4=h5eee18b_1
+  - libxcb=1.15=h7f8727e_0
+  - libxml2=2.9.14=h74e7548_0
+  - lz4-c=1.9.4=h6a678d5_0
+  - mkl=2023.1.0=h6d00ec8_46342
+  - mkl-service=2.4.0=py39h5eee18b_1
+  - mkl_fft=1.3.6=py39h417a72b_1
+  - mkl_random=1.2.2=py39h417a72b_1
+  - monotonic=1.5=py_0
+  - mpc=1.1.0=h10f8cd9_1
+  - mpfr=4.0.2=hb69a4c5_1
+  - mpmath=1.3.0=py39h06a4308_0
+  - multidict=6.0.4=py39h5eee18b_0
+  - multiprocess=0.70.15=py39h06a4308_0
+  - ncurses=6.4=h6a678d5_0
+  - nettle=3.7.3=hbbd107a_1
+  - networkx=3.1=py39h06a4308_0
+  - ninja-base=1.10.2=hd09550d_5
+  - nsight-compute=2022.3.0.22=0
+  - numexpr=2.8.4=py39hc78ab66_1
+  - numpy-base=1.25.0=py39hb5e798b_0
+  - oauth2client=4.1.3=py_0
+  - openh264=2.1.1=h4ff587b_0
+  - openjpeg=2.4.0=h3ad879b_0
+  - openssl=1.1.1w=h7f8727e_0
+  - orc=1.7.4=hb3bc3d3_1
+  - pango=1.50.7=h05da053_0
+  - pcre=8.45=h295c915_0
+  - pcre2=10.37=he7ceb23_1
+  - perl=5.34.0=h5eee18b_2
+  - pip=23.0.1=py39h06a4308_0
+  - pixman=0.40.0=h7f8727e_1
+  - poppler=0.81.0=h01f5e8b_2
+  - poppler-data=0.4.11=h06a4308_1
+  - pyarrow=11.0.0=py39h468efa6_1
+  - pycparser=2.21=pyhd3eb1b0_0
+  - pyopenssl=23.2.0=py39h06a4308_0
+  - pyparsing=3.0.9=py39h06a4308_0
+  - pysocks=1.7.1=py39h06a4308_0
+  - python=3.9.16=h7a1cb2a_2
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python-devtools=0.11.0=pyhd8ed1ab_0
+  - python-graphviz=0.20.1=py39h06a4308_0
+  - python-tzdata=2023.3=pyhd3eb1b0_0
+  - python-xxhash=2.0.2=py39h5eee18b_1
+  - python_abi=3.9=2_cp39
+  - pytorch=2.0.1=py3.9_cuda11.8_cudnn8.7.0_0
+  - pytorch-cuda=11.8=h7e8668a_5
+  - pytorch-mutex=1.0=cuda
+  - pytz=2023.3=pyhd8ed1ab_0
+  - pyu2f=0.1.5=pyhd8ed1ab_0
+  - pyyaml=6.0=py39hb9d737c_4
+  - re2=2022.04.01=h27087fc_0
+  - readline=8.2=h5eee18b_0
+  - retry_decorator=1.1.1=pyh9f0ad1d_0
+  - sacremoses=0.0.53=pyhd8ed1ab_0
+  - setuptools=66.0.0=py39h06a4308_0
+  - six=1.16.0=pyh6c4a22f_0
+  - snappy=1.1.9=h295c915_0
+  - sqlite=3.41.2=h5eee18b_0
+  - sympy=1.11.1=py39h06a4308_0
+  - tbb=2021.8.0=hdb19cb5_0
+  - tk=8.6.12=h1ccaba5_0
+  - tmux=3.2a=h385fc29_0
+  - torchaudio=2.0.2=py39_cu118
+  - torchtriton=2.0.0=py39
+  - typing_extensions=4.4.0=py39h06a4308_0
+  - tzdata=2024a=h0c530f3_0
+  - utf8proc=2.6.1=h27cfd23_0
+  - wheel=0.38.4=py39h06a4308_0
+  - xz=5.2.10=h5eee18b_1
+  - yaml=0.2.5=h7f98852_2
+  - zlib=1.2.13=h5eee18b_0
+  - zstd=1.5.5=hc292b87_0
+  - pip:
+      - absl-py==2.0.0
+      - accelerate==0.16.0
+      - aiofiles==23.1.0
+      - aiohttp==3.8.4
+      - albumentations==1.3.1
+      - altair==4.2.2
+      - antlr4-python3-runtime==4.9.3
+      - anyio==3.6.2
+      - appdirs==1.4.4
+      - apptools==5.2.1
+      - argon2-cffi==21.3.0
+      - argon2-cffi-bindings==21.2.0
+      - argparse==1.4.0
+      - arrow==1.2.3
+      - attrs==22.2.0
+      - backcall==0.2.0
+      - batchgenerators==0.25
+      - beautifulsoup4==4.12.2
+      - bitsandbytes==0.37.0
+      - bitsandbytes-cuda117==0.26.0.post2
+      - bleach==6.0.0
+      - blis==0.7.9
+      - braceexpand==0.1.7
+      - brotli==1.1.0
+      - cachetools==5.3.1
+      - catalogue==2.0.8
+      - cchardet==2.1.7
+      - chardet==5.1.0
+      - charset-normalizer==3.1.0
+      - click==8.1.3
+      - cmake==3.26.3
+      - comm==0.1.3
+      - commonmark==0.9.1
+      - conda-pack==0.6.0
+      - confection==0.0.4
+      - configobj==5.0.8
+      - conllu==4.5.3
+      - contourpy==1.0.7
+      - cpufeature==0.2.1
+      - cycler==0.11.0
+      - cymem==2.0.7
+      - debugpy==1.6.7
+      - decorator==5.1.1
+      - decord==0.6.0
+      - defusedxml==0.7.1
+      - deprecated==1.2.14
+      - docker-pycreds==0.4.0
+      - efficientnet-pytorch==0.7.1
+      - einops==0.6.1
+      - einops-exts==0.0.4
+      - entrypoints==0.4
+      - envisage==7.0.3
+      - et-xmlfile==1.1.0
+      - exceptiongroup==1.2.0
+      - fairscale==0.4.13
+      - fastapi==0.95.1
+      - fastjsonschema==2.16.3
+      - ffmpy==0.3.0
+      - fonttools==4.38.0
+      - fqdn==1.5.1
+      - fschat==0.1.10
+      - ftfy==6.1.1
+      - future==0.18.3
+      - gitdb==4.0.10
+      - gitpython==3.1.31
+      - google-auth==2.29.0
+      - google-auth-oauthlib==1.0.0
+      - gradio==3.23.0
+      - gradio-client==0.0.8
+      - grpcio==1.59.0
+      - h11==0.14.0
+      - h5py==3.9.0
+      - hjson==3.1.0
+      - httpcore==0.17.0
+      - httpx==0.24.0
+      - huggingface-hub==0.22.2
+      - humanize==4.8.0
+      - hyperlink==21.0.0
+      - imageio==2.33.0
+      - importlib-metadata==6.6.0
+      - importlib-resources==5.12.0
+      - inflate64==1.0.0
+      - iniconfig==2.0.0
+      - inquirerpy==0.3.4
+      - iopath==0.1.10
+      - ipdb==0.13.13
+      - ipykernel==6.22.0
+      - ipython==8.12.0
+      - ipython-genutils==0.2.0
+      - isoduration==20.11.0
+      - jedi==0.18.2
+      - jsonpointer==2.3
+      - jsonschema==4.17.3
+      - jupyter-client==8.2.0
+      - jupyter-core==5.3.0
+      - jupyter-events==0.6.3
+      - jupyter-server==2.5.0
+      - jupyter-server-terminals==0.4.4
+      - jupyterlab-pygments==0.2.2
+      - kiwisolver==1.4.4
+      - langcodes==3.3.0
+      - lazy-loader==0.3
+      - linecache2==1.0.0
+      - linkify-it-py==2.0.0
+      - lit==16.0.2
+      - llvmlite==0.39.1
+      - markdown==3.5
+      - markdown-it-py==2.2.0
+      - markdown2==2.4.8
+      - markupsafe==2.1.2
+      - matplotlib==3.7.0
+      - matplotlib-inline==0.1.6
+      - mdit-py-plugins==0.3.3
+      - mdurl==0.1.2
+      - mistune==2.0.5
+      - multivolumefile==0.2.3
+      - munch==4.0.0
+      - murmurhash==1.0.9
+      - mypy-extensions==1.0.0
+      - nbclassic==0.5.6
+      - nbclient==0.7.4
+      - nbconvert==7.3.1
+      - nbformat==5.8.0
+      - nest-asyncio==1.5.6
+      - nh3==0.2.11
+      - nibabel==5.1.0
+      - ninja==1.11.1
+      - nltk==3.8.1
+      - nmslib==2.1.1
+      - notebook==6.5.4
+      - notebook-shim==0.2.3
+      - numba==0.56.4
+      - numpy==1.23.5
+      - nvidia-cufft-cu11==10.9.0.58
+      - nvidia-curand-cu11==10.2.10.91
+      - nvidia-cusolver-cu11==11.4.0.1
+      - nvidia-cusparse-cu11==11.7.4.91
+      - nvidia-nccl-cu11==2.14.3
+      - nvidia-nvtx-cu11==11.7.91
+      - oauthlib==3.2.2
+      - omegaconf==2.3.0
+      - open-clip-torch==2.20.0
+      - openai==0.27.0
+      - opencv-contrib-python==4.10.0.82
+      - openpyxl==3.1.2
+      - orjson==3.8.11
+      - packaging==23.0
+      - pandas==2.0.1
+      - pandocfilters==1.5.0
+      - parso==0.8.3
+      - pathtools==0.1.2
+      - pathy==0.10.1
+      - peft==0.2.0
+      - pexpect==4.8.0
+      - pfzy==0.3.4
+      - pickleshare==0.7.5
+      - pillow==9.5.0
+      - platformdirs==3.5.0
+      - pluggy==1.3.0
+      - portalocker==2.7.0
+      - preshed==3.0.8
+      - pretrainedmodels==0.7.4
+      - prometheus-client==0.16.0
+      - prompt-toolkit==3.0.38
+      - protobuf==3.20.3
+      - psutil==5.9.4
+      - ptyprocess==0.7.0
+      - pure-eval==0.2.2
+      - py-cpuinfo==9.0.0
+      - py-rsync==0.0.1a0.dev0
+      - py7zr==0.20.8
+      - pyasn1==0.5.0
+      - pyasn1-modules==0.3.0
+      - pybcj==1.0.2
+      - pybind11==2.6.1
+      - pycocoevalcap==1.2
+      - pycocotools==2.0.6
+      - pycryptodomex==3.19.1
+      - pydantic==1.10.7
+      - pydub==0.25.1
+      - pyface==8.0.0
+      - pygments==2.15.1
+      - pynndescent==0.5.10
+      - pyppmd==1.1.0
+      - pyqt5==5.15.10
+      - pyqt5-qt5==5.15.2
+      - pyqt5-sip==12.13.0
+      - pyrsistent==0.19.3
+      - pysbd==0.3.4
+      - pytest==7.4.3
+      - python-json-logger==2.0.7
+      - python-multipart==0.0.6
+      - python-polylabel==0.6
+      - python-rsync==0.1.0
+      - pyzmq==25.0.2
+      - pyzstd==0.15.9
+      - qudida==0.0.4
+      - regex==2022.10.31
+      - requests==2.29.0
+      - requests-oauthlib==1.3.1
+      - rfc3339-validator==0.1.4
+      - rfc3986-validator==0.1.1
+      - rich==12.6.0
+      - rsa==4.9
+      - safetensors==0.4.3
+      - scikit-image==0.22.0
+      - scikit-learn==1.2.2
+      - scipy==1.10.1
+      - scispacy==0.5.2
+      - segmentation-models-pytorch==0.3.3
+      - semantic-version==2.10.0
+      - send2trash==1.8.2
+      - sentence-transformers==2.2.2
+      - sentencepiece==0.1.98
+      - sentry-sdk==1.21.0
+      - setproctitle==1.3.2
+      - shapely==2.0.2
+      - shellingham==1.5.4
+      - shortuuid==1.0.11
+      - simpleitk==2.2.1
+      - smart-open==6.3.0
+      - smmap==5.0.0
+      - sniffio==1.3.0
+      - soupsieve==2.4.1
+      - spacy==3.4.4
+      - spacy-legacy==3.0.12
+      - spacy-loggers==1.0.4
+      - srsly==2.4.6
+      - stack-data==0.6.2
+      - starlette==0.26.1
+      - surface-distance-based-measures==0.1
+      - svgwrite==1.4.3
+      - swig==4.1.1
+      - tenacity==8.2.2
+      - tensorboard==2.14.1
+      - tensorboard-data-server==0.7.1
+      - terminado==0.17.1
+      - texttable==1.7.0
+      - thinc==8.1.9
+      - threadpoolctl==3.1.0
+      - tifffile==2023.9.26
+      - timm==0.9.2
+      - tinycss2==1.2.1
+      - tokenizers==0.19.1
+      - tomli==2.0.1
+      - toolz==0.12.0
+      - torchio==0.19.2
+      - torchvision==0.15.2
+      - tornado==6.3.1
+      - tqdm==4.64.1
+      - traceback2==1.4.0
+      - traitlets==5.9.0
+      - traits==6.4.3
+      - traitsui==8.0.0
+      - transformers==4.40.0
+      - typer==0.7.0
+      - typing-extensions==4.5.0
+      - typing-inspect==0.8.0
+      - uc-micro-py==1.0.1
+      - umap-learn==0.5.3
+      - unittest2==1.1.0
+      - unzip==1.0.0
+      - uri-template==1.2.0
+      - urllib3==1.26.15
+      - uvicorn==0.22.0
+      - vtk==9.3.0
+      - wandb==0.15.0
+      - wasabi==0.10.1
+      - wavedrom==2.0.3.post3
+      - wcwidth==0.2.6
+      - webcolors==1.13
+      - webdataset==0.2.48
+      - webencodings==0.5.1
+      - websocket-client==1.5.1
+      - websockets==11.0.2
+      - werkzeug==3.0.0
+      - wrapt==1.16.0
+      - xxhash==3.3.0
+      - yarl==1.8.2
+      - zipp==3.14.0
+prefix: /home/zhouhy/anaconda3/envs/medversa

medomni/models/medomni.py

@@ -285,7 +285,6 @@ class MedOmni(Blip2Base, PyTorchModelHubMixin):
         if self.low_resource:
             self.vit_to_cpu()
             image = image.to("cpu")
-        ipdb.set_trace()
         with self.maybe_autocast(device):
             if 'ct' in modals:
                 image_embeds_list = self.visual_encoder_3d(image, encoder_only=True)
@@ -484,20 +483,6 @@ class MedOmni(Blip2Base, PyTorchModelHubMixin):
 
     @classmethod
     def from_config(cls, cfg, finetune=False):
-        # llama_model = cfg.get("llama_model")
-        # freeze_vit = cfg.get("freeze_vit", True)
-        # low_resource = cfg.get("low_resource", False)
-        # max_txt_len = cfg.get("max_txt_len", 32)
-        # end_sym = cfg.get("end_sym", '\n')
-        # ipdb.set_trace()
-
-        # model = cls(
-        #     freeze_vit=freeze_vit,
-        #     llama_model=llama_model,
-        #     max_txt_len=max_txt_len,
-        #     low_resource=low_resource,
-        #     end_sym=end_sym
-        # )
         model = cls(cfg)
 
         # load checkpoint