Llava Next Example

Source vllm-project/vllm.

  1.  1from io import BytesIO
  2.  2
  3.  3import requests
  4.  4from PIL import Image
  5.  5
  6.  6from vllm import LLM, SamplingParams
  7.  7
  8.  8
  9.  9def run_llava_next():
  10. 10    llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf", max_model_len=4096)
  11. 11
  12. 12    prompt = "[INST] <image>\nWhat is shown in this image? [/INST]"
  13. 13    url = "https://h2o-release.s3.amazonaws.com/h2ogpt/bigben.jpg"
  14. 14    image = Image.open(BytesIO(requests.get(url).content))
  15. 15    sampling_params = SamplingParams(temperature=0.8,
  16. 16                                     top_p=0.95,
  17. 17                                     max_tokens=100)
  18. 18
  19. 19    outputs = llm.generate(
  20. 20        {
  21. 21            "prompt": prompt,
  22. 22            "multi_modal_data": {
  23. 23                "image": image
  24. 24            }
  25. 25        },
  26. 26        sampling_params=sampling_params)
  27. 27
  28. 28    generated_text = ""
  29. 29    for o in outputs:
  30. 30        generated_text += o.outputs[0].text
  31. 31
  32. 32    print(f"LLM output:{generated_text}")
  33. 33
  34. 34
  35. 35if __name__ == "__main__":
  36. 36    run_llava_next()