Production Metrics

vLLM exposes a number of metrics that can be used to monitor the health of the system. These metrics are exposed via the /metrics endpoint on the vLLM OpenAI compatible API server.

The following metrics are exposed:

  1. class Metrics:
  3.     def __init__(self, labelnames: List[str]):
  4.         # Unregister any existing vLLM collectors
  5.         for collector in list(REGISTRY._collector_to_names):
  6.             if hasattr(collector, "_name") and "vllm" in collector._name:
  7.                 REGISTRY.unregister(collector)
  9.         # Config Information
  10.         self.info_cache_config = Info(
  11.             name='vllm:cache_config',
  12.             documentation='information of cache_config')
  14.         # System stats
  15.         self.gauge_scheduler_running = Gauge(
  16.             name="vllm:num_requests_running",
  17.             documentation="Number of requests currently running on GPU.",
  18.             labelnames=labelnames)
  19.         self.gauge_scheduler_swapped = Gauge(
  20.             name="vllm:num_requests_swapped",
  21.             documentation="Number of requests swapped to CPU.",
  22.             labelnames=labelnames)
  23.         self.gauge_scheduler_waiting = Gauge(
  24.             name="vllm:num_requests_waiting",
  25.             documentation="Number of requests waiting to be processed.",
  26.             labelnames=labelnames)
  27.         self.gauge_gpu_cache_usage = Gauge(
  28.             name="vllm:gpu_cache_usage_perc",
  29.             documentation="GPU KV-cache usage. 1 means 100 percent usage.",
  30.             labelnames=labelnames)
  31.         self.gauge_cpu_cache_usage = Gauge(
  32.             name="vllm:cpu_cache_usage_perc",
  33.             documentation="CPU KV-cache usage. 1 means 100 percent usage.",
  34.             labelnames=labelnames)
  36.         # Raw stats from last model iteration
  37.         self.counter_prompt_tokens = Counter(
  38.             name="vllm:prompt_tokens_total",
  39.             documentation="Number of prefill tokens processed.",
  40.             labelnames=labelnames)
  41.         self.counter_generation_tokens = Counter(
  42.             name="vllm:generation_tokens_total",
  43.             documentation="Number of generation tokens processed.",
  44.             labelnames=labelnames)
  45.         self.histogram_time_to_first_token = Histogram(
  46.             name="vllm:time_to_first_token_seconds",
  47.             documentation="Histogram of time to first token in seconds.",
  48.             labelnames=labelnames,
  49.             buckets=[
  50.                 0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
  51.                 0.75, 1.0, 2.5, 5.0, 7.5, 10.0
  52.             ])
  53.         self.histogram_time_per_output_token = Histogram(
  54.             name="vllm:time_per_output_token_seconds",
  55.             documentation="Histogram of time per output token in seconds.",
  56.             labelnames=labelnames,
  57.             buckets=[
  58.                 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
  59.                 1.0, 2.5
  60.             ])
  61.         self.histogram_e2e_request_latency = Histogram(
  62.             name="vllm:e2e_request_latency_seconds",
  63.             documentation="Histogram of end to end request latency in seconds.",
  64.             labelnames=labelnames,
  65.             buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
  67.         # Legacy metrics
  68.         self.gauge_avg_prompt_throughput = Gauge(
  69.             name="vllm:avg_prompt_throughput_toks_per_s",
  70.             documentation="Average prefill throughput in tokens/s.",
  71.             labelnames=labelnames,
  72.         )
  73.         self.gauge_avg_generation_throughput = Gauge(
  74.             name="vllm:avg_generation_throughput_toks_per_s",
  75.             documentation="Average generation throughput in tokens/s.",
  76.             labelnames=labelnames,
  77.         )