Production Metrics

vLLM exposes a number of metrics that can be used to monitor the health of the system. These metrics are exposed via the /metrics endpoint on the vLLM OpenAI compatible API server.

You can start the server using Python, or using Docker:

  1. vllm serve unsloth/Llama-3.2-1B-Instruct

Then query the endpoint to get the latest metrics from the server:

  1. $ curl http://0.0.0.0:8000/metrics
  3. # HELP vllm:iteration_tokens_total Histogram of number of tokens per engine_step.
  4. # TYPE vllm:iteration_tokens_total histogram
  5. vllm:iteration_tokens_total_sum{model_name="unsloth/Llama-3.2-1B-Instruct"} 0.0
  6. vllm:iteration_tokens_total_bucket{le="1.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
  7. vllm:iteration_tokens_total_bucket{le="8.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
  8. vllm:iteration_tokens_total_bucket{le="16.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
  9. vllm:iteration_tokens_total_bucket{le="32.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
  10. vllm:iteration_tokens_total_bucket{le="64.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
  11. vllm:iteration_tokens_total_bucket{le="128.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
  12. vllm:iteration_tokens_total_bucket{le="256.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
  13. vllm:iteration_tokens_total_bucket{le="512.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
  14. ...

The following metrics are exposed:

  1. class Metrics:
  2.     """
  3.     vLLM uses a multiprocessing-based frontend for the OpenAI server.
  4.     This means that we need to run prometheus_client in multiprocessing mode
  5.     See https://prometheus.github.io/client_python/multiprocess/ for more
  6.     details on limitations.
  7.     """
  9.     labelname_finish_reason = "finished_reason"
  10.     labelname_waiting_lora_adapters = "waiting_lora_adapters"
  11.     labelname_running_lora_adapters = "running_lora_adapters"
  12.     labelname_max_lora = "max_lora"
  13.     _gauge_cls = prometheus_client.Gauge
  14.     _counter_cls = prometheus_client.Counter
  15.     _histogram_cls = prometheus_client.Histogram
  17.     def __init__(self, labelnames: List[str], vllm_config: VllmConfig):
  18.         # Unregister any existing vLLM collectors (for CI/CD)
  19.         self._unregister_vllm_metrics()
  21.         max_model_len = vllm_config.model_config.max_model_len
  23.         # System stats
  24.         #   Scheduler State
  25.         self.gauge_scheduler_running = self._gauge_cls(
  26.             name="vllm:num_requests_running",
  27.             documentation="Number of requests currently running on GPU.",
  28.             labelnames=labelnames,
  29.             multiprocess_mode="sum")
  30.         self.gauge_scheduler_waiting = self._gauge_cls(
  31.             name="vllm:num_requests_waiting",
  32.             documentation="Number of requests waiting to be processed.",
  33.             labelnames=labelnames,
  34.             multiprocess_mode="sum")
  35.         self.gauge_lora_info = self._gauge_cls(
  36.             name="vllm:lora_requests_info",
  37.             documentation="Running stats on lora requests.",
  38.             labelnames=[
  39.                 self.labelname_running_lora_adapters,
  40.                 self.labelname_max_lora,
  41.                 self.labelname_waiting_lora_adapters,
  42.             ],
  43.             multiprocess_mode="livemostrecent",
  44.         )
  45.         self.gauge_scheduler_swapped = self._gauge_cls(
  46.             name="vllm:num_requests_swapped",
  47.             documentation="Number of requests swapped to CPU.",
  48.             labelnames=labelnames,
  49.             multiprocess_mode="sum")
  50.         #   KV Cache Usage in %
  51.         self.gauge_gpu_cache_usage = self._gauge_cls(
  52.             name="vllm:gpu_cache_usage_perc",
  53.             documentation="GPU KV-cache usage. 1 means 100 percent usage.",
  54.             labelnames=labelnames,
  55.             multiprocess_mode="sum")
  56.         self.gauge_cpu_cache_usage = self._gauge_cls(
  57.             name="vllm:cpu_cache_usage_perc",
  58.             documentation="CPU KV-cache usage. 1 means 100 percent usage.",
  59.             labelnames=labelnames,
  60.             multiprocess_mode="sum")
  61.         #   Prefix caching block hit rate
  62.         self.gauge_cpu_prefix_cache_hit_rate = self._gauge_cls(
  63.             name="vllm:cpu_prefix_cache_hit_rate",
  64.             documentation="CPU prefix cache block hit rate.",
  65.             labelnames=labelnames,
  66.             multiprocess_mode="sum")
  67.         self.gauge_gpu_prefix_cache_hit_rate = self._gauge_cls(
  68.             name="vllm:gpu_prefix_cache_hit_rate",
  69.             documentation="GPU prefix cache block hit rate.",
  70.             labelnames=labelnames,
  71.             multiprocess_mode="sum")
  73.         # Iteration stats
  74.         self.counter_num_preemption = self._counter_cls(
  75.             name="vllm:num_preemptions_total",
  76.             documentation="Cumulative number of preemption from the engine.",
  77.             labelnames=labelnames)
  78.         self.counter_prompt_tokens = self._counter_cls(
  79.             name="vllm:prompt_tokens_total",
  80.             documentation="Number of prefill tokens processed.",
  81.             labelnames=labelnames)
  82.         self.counter_generation_tokens = self._counter_cls(
  83.             name="vllm:generation_tokens_total",
  84.             documentation="Number of generation tokens processed.",
  85.             labelnames=labelnames)
  86.         self.counter_tokens = self._counter_cls(
  87.             name="vllm:tokens_total",
  88.             documentation="Number of prefill plus generation tokens processed.",
  89.             labelnames=labelnames)
  90.         buckets = [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096]
  91.         if not vllm_config.model_config.enforce_eager:
  92.             buckets = vllm_config.compilation_config.\
  93.                 cudagraph_capture_sizes.copy()
  94.             buckets.sort()
  95.         self.histogram_iteration_tokens = self._histogram_cls(
  96.             name="vllm:iteration_tokens_total",
  97.             documentation="Histogram of number of tokens per engine_step.",
  98.             labelnames=labelnames,
  99.             buckets=buckets)
  100.         self.histogram_time_to_first_token = self._histogram_cls(
  101.             name="vllm:time_to_first_token_seconds",
  102.             documentation="Histogram of time to first token in seconds.",
  103.             labelnames=labelnames,
  104.             buckets=[
  105.                 0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
  106.                 0.75, 1.0, 2.5, 5.0, 7.5, 10.0
  107.             ])
  108.         self.histogram_time_per_output_token = self._histogram_cls(
  109.             name="vllm:time_per_output_token_seconds",
  110.             documentation="Histogram of time per output token in seconds.",
  111.             labelnames=labelnames,
  112.             buckets=[
  113.                 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
  114.                 1.0, 2.5
  115.             ])
  117.         # Request stats
  118.         #   Latency
  119.         request_latency_buckets = [
  120.             0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
  121.             40.0, 50.0, 60.0
  122.         ]
  123.         self.histogram_e2e_time_request = self._histogram_cls(
  124.             name="vllm:e2e_request_latency_seconds",
  125.             documentation="Histogram of end to end request latency in seconds.",
  126.             labelnames=labelnames,
  127.             buckets=request_latency_buckets)
  128.         self.histogram_queue_time_request = self._histogram_cls(
  129.             name="vllm:request_queue_time_seconds",
  130.             documentation=
  131.             "Histogram of time spent in WAITING phase for request.",
  132.             labelnames=labelnames,
  133.             buckets=request_latency_buckets)
  134.         self.histogram_inference_time_request = self._histogram_cls(
  135.             name="vllm:request_inference_time_seconds",
  136.             documentation=
  137.             "Histogram of time spent in RUNNING phase for request.",
  138.             labelnames=labelnames,
  139.             buckets=request_latency_buckets)
  140.         self.histogram_prefill_time_request = self._histogram_cls(
  141.             name="vllm:request_prefill_time_seconds",
  142.             documentation=
  143.             "Histogram of time spent in PREFILL phase for request.",
  144.             labelnames=labelnames,
  145.             buckets=request_latency_buckets)
  146.         self.histogram_decode_time_request = self._histogram_cls(
  147.             name="vllm:request_decode_time_seconds",
  148.             documentation=
  149.             "Histogram of time spent in DECODE phase for request.",
  150.             labelnames=labelnames,
  151.             buckets=request_latency_buckets)
  152.         self.histogram_time_in_queue_request = self._histogram_cls(
  153.             name="vllm:time_in_queue_requests",
  154.             documentation=
  155.             "Histogram of time the request spent in the queue in seconds.",
  156.             labelnames=labelnames,
  157.             buckets=request_latency_buckets)
  158.         self.histogram_model_forward_time_request = self._histogram_cls(
  159.             name="vllm:model_forward_time_milliseconds",
  160.             documentation=
  161.             "Histogram of time spent in the model forward pass in ms.",
  162.             labelnames=labelnames,
  163.             buckets=build_1_2_3_5_8_buckets(3000))
  164.         self.histogram_model_execute_time_request = self._histogram_cls(
  165.             name="vllm:model_execute_time_milliseconds",
  166.             documentation=
  167.             "Histogram of time spent in the model execute function in ms.",
  168.             labelnames=labelnames,
  169.             buckets=build_1_2_3_5_8_buckets(3000))
  170.         #   Metadata
  171.         self.histogram_num_prompt_tokens_request = self._histogram_cls(
  172.             name="vllm:request_prompt_tokens",
  173.             documentation="Number of prefill tokens processed.",
  174.             labelnames=labelnames,
  175.             buckets=build_1_2_5_buckets(max_model_len),
  176.         )
  177.         self.histogram_num_generation_tokens_request = \
  178.             self._histogram_cls(
  179.                 name="vllm:request_generation_tokens",
  180.                 documentation="Number of generation tokens processed.",
  181.                 labelnames=labelnames,
  182.                 buckets=build_1_2_5_buckets(max_model_len),
  183.             )
  184.         self.histogram_max_num_generation_tokens_request = self._histogram_cls(
  185.             name="vllm:request_max_num_generation_tokens",
  186.             documentation=
  187.             "Histogram of maximum number of requested generation tokens.",
  188.             labelnames=labelnames,
  189.             buckets=build_1_2_5_buckets(max_model_len))
  190.         self.histogram_n_request = self._histogram_cls(
  191.             name="vllm:request_params_n",
  192.             documentation="Histogram of the n request parameter.",
  193.             labelnames=labelnames,
  194.             buckets=[1, 2, 5, 10, 20],
  195.         )
  196.         self.histogram_max_tokens_request = self._histogram_cls(
  197.             name="vllm:request_params_max_tokens",
  198.             documentation="Histogram of the max_tokens request parameter.",
  199.             labelnames=labelnames,
  200.             buckets=build_1_2_5_buckets(max_model_len),
  201.         )
  202.         self.counter_request_success = self._counter_cls(
  203.             name="vllm:request_success_total",
  204.             documentation="Count of successfully processed requests.",
  205.             labelnames=labelnames + [Metrics.labelname_finish_reason])
  207.         # Speculatie decoding stats
  208.         self.gauge_spec_decode_draft_acceptance_rate = self._gauge_cls(
  209.             name="vllm:spec_decode_draft_acceptance_rate",
  210.             documentation="Speulative token acceptance rate.",
  211.             labelnames=labelnames,
  212.             multiprocess_mode="sum")
  213.         self.gauge_spec_decode_efficiency = self._gauge_cls(
  214.             name="vllm:spec_decode_efficiency",
  215.             documentation="Speculative decoding system efficiency.",
  216.             labelnames=labelnames,
  217.             multiprocess_mode="sum")
  218.         self.counter_spec_decode_num_accepted_tokens = (self._counter_cls(
  219.             name="vllm:spec_decode_num_accepted_tokens_total",
  220.             documentation="Number of accepted tokens.",
  221.             labelnames=labelnames))
  222.         self.counter_spec_decode_num_draft_tokens = self._counter_cls(
  223.             name="vllm:spec_decode_num_draft_tokens_total",
  224.             documentation="Number of draft tokens.",
  225.             labelnames=labelnames)
  226.         self.counter_spec_decode_num_emitted_tokens = (self._counter_cls(
  227.             name="vllm:spec_decode_num_emitted_tokens_total",
  228.             documentation="Number of emitted tokens.",
  229.             labelnames=labelnames))