Production Metrics

vLLM exposes a number of metrics that can be used to monitor the health of the system. These metrics are exposed via the /metrics endpoint on the vLLM OpenAI compatible API server.

The following metrics are exposed:

  1. class Metrics:
  2.     """
  3.     vLLM uses a multiprocessing-based frontend for the OpenAI server.
  4.     This means that we need to run prometheus_client in multiprocessing mode
  5.     See https://prometheus.github.io/client_python/multiprocess/ for more
  6.     details on limitations.
  7.     """
  8.     labelname_finish_reason = "finished_reason"
  9.     _gauge_cls = prometheus_client.Gauge
  10.     _counter_cls = prometheus_client.Counter
  11.     _histogram_cls = prometheus_client.Histogram
  13.     def __init__(self, labelnames: List[str], max_model_len: int):
  14.         # Unregister any existing vLLM collectors (for CI/CD)
  15.         self._unregister_vllm_metrics()
  17.         # System stats
  18.         #   Scheduler State
  19.         self.gauge_scheduler_running = self._gauge_cls(
  20.             name="vllm:num_requests_running",
  21.             documentation="Number of requests currently running on GPU.",
  22.             labelnames=labelnames,
  23.             multiprocess_mode="sum")
  24.         self.gauge_scheduler_waiting = self._gauge_cls(
  25.             name="vllm:num_requests_waiting",
  26.             documentation="Number of requests waiting to be processed.",
  27.             labelnames=labelnames,
  28.             multiprocess_mode="sum")
  29.         self.gauge_scheduler_swapped = self._gauge_cls(
  30.             name="vllm:num_requests_swapped",
  31.             documentation="Number of requests swapped to CPU.",
  32.             labelnames=labelnames,
  33.             multiprocess_mode="sum")
  34.         #   KV Cache Usage in %
  35.         self.gauge_gpu_cache_usage = self._gauge_cls(
  36.             name="vllm:gpu_cache_usage_perc",
  37.             documentation="GPU KV-cache usage. 1 means 100 percent usage.",
  38.             labelnames=labelnames,
  39.             multiprocess_mode="sum")
  40.         self.gauge_cpu_cache_usage = self._gauge_cls(
  41.             name="vllm:cpu_cache_usage_perc",
  42.             documentation="CPU KV-cache usage. 1 means 100 percent usage.",
  43.             labelnames=labelnames,
  44.             multiprocess_mode="sum")
  45.         #   Prefix caching block hit rate
  46.         self.gauge_cpu_prefix_cache_hit_rate = self._gauge_cls(
  47.             name="vllm:cpu_prefix_cache_hit_rate",
  48.             documentation="CPU prefix cache block hit rate.",
  49.             labelnames=labelnames,
  50.             multiprocess_mode="sum")
  51.         self.gauge_gpu_prefix_cache_hit_rate = self._gauge_cls(
  52.             name="vllm:gpu_prefix_cache_hit_rate",
  53.             documentation="GPU prefix cache block hit rate.",
  54.             labelnames=labelnames,
  55.             multiprocess_mode="sum")
  57.         # Iteration stats
  58.         self.counter_num_preemption = self._counter_cls(
  59.             name="vllm:num_preemptions_total",
  60.             documentation="Cumulative number of preemption from the engine.",
  61.             labelnames=labelnames)
  62.         self.counter_prompt_tokens = self._counter_cls(
  63.             name="vllm:prompt_tokens_total",
  64.             documentation="Number of prefill tokens processed.",
  65.             labelnames=labelnames)
  66.         self.counter_generation_tokens = self._counter_cls(
  67.             name="vllm:generation_tokens_total",
  68.             documentation="Number of generation tokens processed.",
  69.             labelnames=labelnames)
  70.         self.histogram_time_to_first_token = self._histogram_cls(
  71.             name="vllm:time_to_first_token_seconds",
  72.             documentation="Histogram of time to first token in seconds.",
  73.             labelnames=labelnames,
  74.             buckets=[
  75.                 0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
  76.                 0.75, 1.0, 2.5, 5.0, 7.5, 10.0
  77.             ])
  78.         self.histogram_time_per_output_token = self._histogram_cls(
  79.             name="vllm:time_per_output_token_seconds",
  80.             documentation="Histogram of time per output token in seconds.",
  81.             labelnames=labelnames,
  82.             buckets=[
  83.                 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
  84.                 1.0, 2.5
  85.             ])
  87.         # Request stats
  88.         #   Latency
  89.         self.histogram_e2e_time_request = self._histogram_cls(
  90.             name="vllm:e2e_request_latency_seconds",
  91.             documentation="Histogram of end to end request latency in seconds.",
  92.             labelnames=labelnames,
  93.             buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
  94.         #   Metadata
  95.         self.histogram_num_prompt_tokens_request = self._histogram_cls(
  96.             name="vllm:request_prompt_tokens",
  97.             documentation="Number of prefill tokens processed.",
  98.             labelnames=labelnames,
  99.             buckets=build_1_2_5_buckets(max_model_len),
  100.         )
  101.         self.histogram_num_generation_tokens_request = \
  102.             self._histogram_cls(
  103.                 name="vllm:request_generation_tokens",
  104.                 documentation="Number of generation tokens processed.",
  105.                 labelnames=labelnames,
  106.                 buckets=build_1_2_5_buckets(max_model_len),
  107.             )
  108.         self.histogram_best_of_request = self._histogram_cls(
  109.             name="vllm:request_params_best_of",
  110.             documentation="Histogram of the best_of request parameter.",
  111.             labelnames=labelnames,
  112.             buckets=[1, 2, 5, 10, 20],
  113.         )
  114.         self.histogram_n_request = self._histogram_cls(
  115.             name="vllm:request_params_n",
  116.             documentation="Histogram of the n request parameter.",
  117.             labelnames=labelnames,
  118.             buckets=[1, 2, 5, 10, 20],
  119.         )
  120.         self.counter_request_success = self._counter_cls(
  121.             name="vllm:request_success_total",
  122.             documentation="Count of successfully processed requests.",
  123.             labelnames=labelnames + [Metrics.labelname_finish_reason])
  125.         # Speculatie decoding stats
  126.         self.gauge_spec_decode_draft_acceptance_rate = self._gauge_cls(
  127.             name="vllm:spec_decode_draft_acceptance_rate",
  128.             documentation="Speulative token acceptance rate.",
  129.             labelnames=labelnames,
  130.             multiprocess_mode="sum")
  131.         self.gauge_spec_decode_efficiency = self._gauge_cls(
  132.             name="vllm:spec_decode_efficiency",
  133.             documentation="Speculative decoding system efficiency.",
  134.             labelnames=labelnames,
  135.             multiprocess_mode="sum")
  136.         self.counter_spec_decode_num_accepted_tokens = (self._counter_cls(
  137.             name="vllm:spec_decode_num_accepted_tokens_total",
  138.             documentation="Number of accepted tokens.",
  139.             labelnames=labelnames))
  140.         self.counter_spec_decode_num_draft_tokens = self._counter_cls(
  141.             name="vllm:spec_decode_num_draft_tokens_total",
  142.             documentation="Number of draft tokens.",
  143.             labelnames=labelnames)
  144.         self.counter_spec_decode_num_emitted_tokens = (self._counter_cls(
  145.             name="vllm:spec_decode_num_emitted_tokens_total",
  146.             documentation="Number of emitted tokens.",
  147.             labelnames=labelnames))
  149.         # Deprecated in favor of vllm:prompt_tokens_total
  150.         self.gauge_avg_prompt_throughput = self._gauge_cls(
  151.             name="vllm:avg_prompt_throughput_toks_per_s",
  152.             documentation="Average prefill throughput in tokens/s.",
  153.             labelnames=labelnames,
  154.             multiprocess_mode="sum",
  155.         )
  156.         # Deprecated in favor of vllm:generation_tokens_total
  157.         self.gauge_avg_generation_throughput = self._gauge_cls(
  158.             name="vllm:avg_generation_throughput_toks_per_s",
  159.             documentation="Average generation throughput in tokens/s.",
  160.             labelnames=labelnames,
  161.             multiprocess_mode="sum",
  162.         )