diff --git a/torchbenchmark/canary_models/stable_diffusion_xl/install.py b/torchbenchmark/canary_models/stable_diffusion_xl/install.py
index 993f3043c8..df3a68308c 100644
--- a/torchbenchmark/canary_models/stable_diffusion_xl/install.py
+++ b/torchbenchmark/canary_models/stable_diffusion_xl/install.py
@@ -16,10 +16,10 @@ def load_model_checkpoint():
 
 
 if __name__ == "__main__":
+    install_diffusers()
     if not "HUGGING_FACE_HUB_TOKEN" in os.environ:
         warnings.warn(
             "Make sure to set `HUGGINGFACE_HUB_TOKEN` so you can download weights"
         )
     else:
-        install_diffusers()
         load_model_checkpoint()