OpenAI Vision API Client

Source vllm-project/vllm.

  1.  1"""An example showing how to use vLLM to serve VLMs.
  2.  2
  3.  3Launch the vLLM server with the following command:
  4.  4vllm serve llava-hf/llava-1.5-7b-hf --chat-template template_llava.jinja
  5.  5"""
  6.  6import base64
  7.  7
  8.  8import requests
  9.  9from openai import OpenAI
  10. 10
  11. 11# Modify OpenAI's API key and API base to use vLLM's API server.
  12. 12openai_api_key = "EMPTY"
  13. 13openai_api_base = "http://localhost:8000/v1"
  14. 14
  15. 15client = OpenAI(
  16. 16    # defaults to os.environ.get("OPENAI_API_KEY")
  17. 17    api_key=openai_api_key,
  18. 18    base_url=openai_api_base,
  19. 19)
  20. 20
  21. 21models = client.models.list()
  22. 22model = models.data[0].id
  23. 23
  24. 24image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
  25. 25
  26. 26# Use image url in the payload
  27. 27chat_completion_from_url = client.chat.completions.create(
  28. 28    messages=[{
  29. 29        "role":
  30. 30        "user",
  31. 31        "content": [
  32. 32            {
  33. 33                "type": "text",
  34. 34                "text": "What’s in this image?"
  35. 35            },
  36. 36            {
  37. 37                "type": "image_url",
  38. 38                "image_url": {
  39. 39                    "url": image_url
  40. 40                },
  41. 41            },
  42. 42        ],
  43. 43    }],
  44. 44    model=model,
  45. 45    max_tokens=64,
  46. 46)
  47. 47
  48. 48result = chat_completion_from_url.choices[0].message.content
  49. 49print(f"Chat completion output:{result}")
  50. 50
  51. 51
  52. 52# Use base64 encoded image in the payload
  53. 53def encode_image_base64_from_url(image_url: str) -> str:
  54. 54    """Encode an image retrieved from a remote url to base64 format."""
  55. 55
  56. 56    with requests.get(image_url) as response:
  57. 57        response.raise_for_status()
  58. 58        result = base64.b64encode(response.content).decode('utf-8')
  59. 59
  60. 60    return result
  61. 61
  62. 62
  63. 63image_base64 = encode_image_base64_from_url(image_url=image_url)
  64. 64chat_completion_from_base64 = client.chat.completions.create(
  65. 65    messages=[{
  66. 66        "role":
  67. 67        "user",
  68. 68        "content": [
  69. 69            {
  70. 70                "type": "text",
  71. 71                "text": "What’s in this image?"
  72. 72            },
  73. 73            {
  74. 74                "type": "image_url",
  75. 75                "image_url": {
  76. 76                    "url": f"data:image/jpeg;base64,{image_base64}"
  77. 77                },
  78. 78            },
  79. 79        ],
  80. 80    }],
  81. 81    model=model,
  82. 82    max_tokens=64,
  83. 83)
  84. 84
  85. 85result = chat_completion_from_base64.choices[0].message.content
  86. 86print(f"Chat completion output:{result}")