cleanup

Author: tlrmchlsmth

examples/offline_inference.py

@@ -1,5 +1,4 @@
 from vllm import LLM, SamplingParams
-import torch
 
 # Sample prompts.
 prompts = [
@@ -9,12 +8,10 @@
     "The future of AI is",
 ]
 # Create a sampling params object.
-sampling_params = SamplingParams(temperature=0.0, top_p=0.95)
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
 
 # Create an LLM.
-#llm = LLM(model="state-spaces/mamba-370m-hf", dtype=torch.float32)
-llm = LLM(model="state-spaces/mamba2-130m", dtype=torch.float32)
-
+llm = LLM(model="facebook/opt-125m")
 # Generate texts from the prompts. The output is a list of RequestOutput objects
 # that contain the prompt, generated text, and other information.
 outputs = llm.generate(prompts, sampling_params)