Production Metrics

vLLM exposes a number of metrics that can be used to monitor the health of the system. These metrics are exposed via the /metrics endpoint on the vLLM OpenAI compatible API server.

The following metrics are exposed:

  1. class Metrics:
  2.     labelname_finish_reason = "finished_reason"
  4.     def __init__(self, labelnames: List[str], max_model_len: int):
  5.         # Unregister any existing vLLM collectors
  6.         for collector in list(REGISTRY._collector_to_names):
  7.             if hasattr(collector, "_name") and "vllm" in collector._name:
  8.                 REGISTRY.unregister(collector)
  10.         # Config Information
  11.         self.info_cache_config = Info(
  12.             name='vllm:cache_config',
  13.             documentation='information of cache_config')
  15.         # System stats
  16.         #   Scheduler State
  17.         self.gauge_scheduler_running = Gauge(
  18.             name="vllm:num_requests_running",
  19.             documentation="Number of requests currently running on GPU.",
  20.             labelnames=labelnames)
  21.         self.gauge_scheduler_waiting = Gauge(
  22.             name="vllm:num_requests_waiting",
  23.             documentation="Number of requests waiting to be processed.",
  24.             labelnames=labelnames)
  25.         self.gauge_scheduler_swapped = Gauge(
  26.             name="vllm:num_requests_swapped",
  27.             documentation="Number of requests swapped to CPU.",
  28.             labelnames=labelnames)
  29.         #   KV Cache Usage in %
  30.         self.gauge_gpu_cache_usage = Gauge(
  31.             name="vllm:gpu_cache_usage_perc",
  32.             documentation="GPU KV-cache usage. 1 means 100 percent usage.",
  33.             labelnames=labelnames)
  34.         self.gauge_cpu_cache_usage = Gauge(
  35.             name="vllm:cpu_cache_usage_perc",
  36.             documentation="CPU KV-cache usage. 1 means 100 percent usage.",
  37.             labelnames=labelnames)
  39.         # Iteration stats
  40.         self.counter_num_preemption = Counter(
  41.             name="vllm:num_preemptions_total",
  42.             documentation="Cumulative number of preemption from the engine.",
  43.             labelnames=labelnames)
  44.         self.counter_prompt_tokens = Counter(
  45.             name="vllm:prompt_tokens_total",
  46.             documentation="Number of prefill tokens processed.",
  47.             labelnames=labelnames)
  48.         self.counter_generation_tokens = Counter(
  49.             name="vllm:generation_tokens_total",
  50.             documentation="Number of generation tokens processed.",
  51.             labelnames=labelnames)
  52.         self.histogram_time_to_first_token = Histogram(
  53.             name="vllm:time_to_first_token_seconds",
  54.             documentation="Histogram of time to first token in seconds.",
  55.             labelnames=labelnames,
  56.             buckets=[
  57.                 0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
  58.                 0.75, 1.0, 2.5, 5.0, 7.5, 10.0
  59.             ])
  60.         self.histogram_time_per_output_token = Histogram(
  61.             name="vllm:time_per_output_token_seconds",
  62.             documentation="Histogram of time per output token in seconds.",
  63.             labelnames=labelnames,
  64.             buckets=[
  65.                 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
  66.                 1.0, 2.5
  67.             ])
  69.         # Request stats
  70.         #   Latency
  71.         self.histogram_e2e_time_request = Histogram(
  72.             name="vllm:e2e_request_latency_seconds",
  73.             documentation="Histogram of end to end request latency in seconds.",
  74.             labelnames=labelnames,
  75.             buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
  76.         #   Metadata
  77.         self.histogram_num_prompt_tokens_request = Histogram(
  78.             name="vllm:request_prompt_tokens",
  79.             documentation="Number of prefill tokens processed.",
  80.             labelnames=labelnames,
  81.             buckets=build_1_2_5_buckets(max_model_len),
  82.         )
  83.         self.histogram_num_generation_tokens_request = Histogram(
  84.             name="vllm:request_generation_tokens",
  85.             documentation="Number of generation tokens processed.",
  86.             labelnames=labelnames,
  87.             buckets=build_1_2_5_buckets(max_model_len),
  88.         )
  89.         self.histogram_best_of_request = Histogram(
  90.             name="vllm:request_params_best_of",
  91.             documentation="Histogram of the best_of request parameter.",
  92.             labelnames=labelnames,
  93.             buckets=[1, 2, 5, 10, 20],
  94.         )
  95.         self.histogram_n_request = Histogram(
  96.             name="vllm:request_params_n",
  97.             documentation="Histogram of the n request parameter.",
  98.             labelnames=labelnames,
  99.             buckets=[1, 2, 5, 10, 20],
  100.         )
  101.         self.counter_request_success = Counter(
  102.             name="vllm:request_success_total",
  103.             documentation="Count of successfully processed requests.",
  104.             labelnames=labelnames + [Metrics.labelname_finish_reason])
  106.         # Deprecated in favor of vllm:prompt_tokens_total
  107.         self.gauge_avg_prompt_throughput = Gauge(
  108.             name="vllm:avg_prompt_throughput_toks_per_s",
  109.             documentation="Average prefill throughput in tokens/s.",
  110.             labelnames=labelnames,
  111.         )
  112.         # Deprecated in favor of vllm:generation_tokens_total
  113.         self.gauge_avg_generation_throughput = Gauge(
  114.             name="vllm:avg_generation_throughput_toks_per_s",
  115.             documentation="Average generation throughput in tokens/s.",
  116.             labelnames=labelnames,
  117.         )