Environment Variables

vLLM uses the following environment variables to configure the system:

Warning

Please note that VLLM_PORT and VLLM_HOST_IP set the port and ip for vLLM’s internal usage. It is not the port and ip for the API server. If you use --host $VLLM_HOST_IP and --port $VLLM_PORT to start the API server, it will not work.

All environment variables used by vLLM are prefixed with VLLM_. Special care should be taken for Kubernetes users: please do not name the service as vllm, otherwise environment variables set by Kubernetes might conflict with vLLM’s environment variables, because Kubernetes sets environment variables for each service with the capitalized service name as the prefix.

  2. environment_variables: Dict[str, Callable[[], Any]] = {
  4.     # ================== Installation Time Env Vars ==================
  6.     # Target device of vLLM, supporting [cuda (by default),
  7.     # rocm, neuron, cpu, openvino]
  8.     "VLLM_TARGET_DEVICE":
  9.     lambda: os.getenv("VLLM_TARGET_DEVICE", "cuda"),
  11.     # Maximum number of compilation jobs to run in parallel.
  12.     # By default this is the number of CPUs
  13.     "MAX_JOBS":
  14.     lambda: os.getenv("MAX_JOBS", None),
  16.     # Number of threads to use for nvcc
  17.     # By default this is 1.
  18.     # If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU.
  19.     "NVCC_THREADS":
  20.     lambda: os.getenv("NVCC_THREADS", None),
  22.     # If set, vllm will use precompiled binaries (*.so)
  23.     "VLLM_USE_PRECOMPILED":
  24.     lambda: bool(os.environ.get("VLLM_USE_PRECOMPILED")),
  26.     # If set, vllm will install Punica kernels
  27.     "VLLM_INSTALL_PUNICA_KERNELS":
  28.     lambda: bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "0"))),
  30.     # CMake build type
  31.     # If not set, defaults to "Debug" or "RelWithDebInfo"
  32.     # Available options: "Debug", "Release", "RelWithDebInfo"
  33.     "CMAKE_BUILD_TYPE":
  34.     lambda: os.getenv("CMAKE_BUILD_TYPE"),
  36.     # If set, vllm will print verbose logs during installation
  37.     "VERBOSE":
  38.     lambda: bool(int(os.getenv('VERBOSE', '0'))),
  40.     # Root directory for VLLM configuration files
  41.     # Defaults to `~/.config/vllm` unless `XDG_CONFIG_HOME` is set
  42.     # Note that this not only affects how vllm finds its configuration files
  43.     # during runtime, but also affects how vllm installs its configuration
  44.     # files during **installation**.
  45.     "VLLM_CONFIG_ROOT":
  46.     lambda: os.path.expanduser(
  47.         os.getenv(
  48.             "VLLM_CONFIG_ROOT",
  49.             os.path.join(get_default_config_root(), "vllm"),
  50.         )),
  52.     # ================== Runtime Env Vars ==================
  54.     # Root directory for VLLM cache files
  55.     # Defaults to `~/.cache/vllm` unless `XDG_CACHE_HOME` is set
  56.     "VLLM_CACHE_ROOT":
  57.     lambda: os.path.expanduser(
  58.         os.getenv(
  59.             "VLLM_CACHE_ROOT",
  60.             os.path.join(get_default_cache_root(), "vllm"),
  61.         )),
  63.     # used in distributed environment to determine the master address
  64.     'VLLM_HOST_IP':
  65.     lambda: os.getenv('VLLM_HOST_IP', "") or os.getenv("HOST_IP", ""),
  67.     # used in distributed environment to manually set the communication port
  68.     # Note: if VLLM_PORT is set, and some code asks for multiple ports, the
  69.     # VLLM_PORT will be used as the first port, and the rest will be generated
  70.     # by incrementing the VLLM_PORT value.
  71.     # '0' is used to make mypy happy
  72.     'VLLM_PORT':
  73.     lambda: int(os.getenv('VLLM_PORT', '0'))
  74.     if 'VLLM_PORT' in os.environ else None,
  76.     # If true, will load models from ModelScope instead of Hugging Face Hub.
  77.     # note that the value is true or false, not numbers
  78.     "VLLM_USE_MODELSCOPE":
  79.     lambda: os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true",
  81.     # Instance id represents an instance of the VLLM. All processes in the same
  82.     # instance should have the same instance id.
  83.     "VLLM_INSTANCE_ID":
  84.     lambda: os.environ.get("VLLM_INSTANCE_ID", None),
  86.     # Interval in seconds to log a warning message when the ring buffer is full
  87.     "VLLM_RINGBUFFER_WARNING_INTERVAL":
  88.     lambda: int(os.environ.get("VLLM_RINGBUFFER_WARNING_INTERVAL", "60")),
  90.     # path to cudatoolkit home directory, under which should be bin, include,
  91.     # and lib directories.
  92.     "CUDA_HOME":
  93.     lambda: os.environ.get("CUDA_HOME", None),
  95.     # Path to the NCCL library file. It is needed because nccl>=2.19 brought
  96.     # by PyTorch contains a bug: https://github.com/NVIDIA/nccl/issues/1234
  97.     "VLLM_NCCL_SO_PATH":
  98.     lambda: os.environ.get("VLLM_NCCL_SO_PATH", None),
  100.     # when `VLLM_NCCL_SO_PATH` is not set, vllm will try to find the nccl
  101.     # library file in the locations specified by `LD_LIBRARY_PATH`
  102.     "LD_LIBRARY_PATH":
  103.     lambda: os.environ.get("LD_LIBRARY_PATH", None),
  105.     # flag to control if vllm should use triton flash attention
  106.     "VLLM_USE_TRITON_FLASH_ATTN":
  107.     lambda: (os.environ.get("VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in
  108.              ("true", "1")),
  110.     # local rank of the process in the distributed setting, used to determine
  111.     # the GPU device id
  112.     "LOCAL_RANK":
  113.     lambda: int(os.environ.get("LOCAL_RANK", "0")),
  115.     # used to control the visible devices in the distributed setting
  116.     "CUDA_VISIBLE_DEVICES":
  117.     lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None),
  119.     # timeout for each iteration in the engine
  120.     "VLLM_ENGINE_ITERATION_TIMEOUT_S":
  121.     lambda: int(os.environ.get("VLLM_ENGINE_ITERATION_TIMEOUT_S", "60")),
  123.     # API key for VLLM API server
  124.     "VLLM_API_KEY":
  125.     lambda: os.environ.get("VLLM_API_KEY", None),
  127.     # S3 access information, used for tensorizer to load model from S3
  128.     "S3_ACCESS_KEY_ID":
  129.     lambda: os.environ.get("S3_ACCESS_KEY_ID", None),
  130.     "S3_SECRET_ACCESS_KEY":
  131.     lambda: os.environ.get("S3_SECRET_ACCESS_KEY", None),
  132.     "S3_ENDPOINT_URL":
  133.     lambda: os.environ.get("S3_ENDPOINT_URL", None),
  135.     # Usage stats collection
  136.     "VLLM_USAGE_STATS_SERVER":
  137.     lambda: os.environ.get("VLLM_USAGE_STATS_SERVER", "https://stats.vllm.ai"),
  138.     "VLLM_NO_USAGE_STATS":
  139.     lambda: os.environ.get("VLLM_NO_USAGE_STATS", "0") == "1",
  140.     "VLLM_DO_NOT_TRACK":
  141.     lambda: (os.environ.get("VLLM_DO_NOT_TRACK", None) or os.environ.get(
  142.         "DO_NOT_TRACK", None) or "0") == "1",
  143.     "VLLM_USAGE_SOURCE":
  144.     lambda: os.environ.get("VLLM_USAGE_SOURCE", "production"),
  146.     # Logging configuration
  147.     # If set to 0, vllm will not configure logging
  148.     # If set to 1, vllm will configure logging using the default configuration
  149.     #    or the configuration file specified by VLLM_LOGGING_CONFIG_PATH
  150.     "VLLM_CONFIGURE_LOGGING":
  151.     lambda: int(os.getenv("VLLM_CONFIGURE_LOGGING", "1")),
  152.     "VLLM_LOGGING_CONFIG_PATH":
  153.     lambda: os.getenv("VLLM_LOGGING_CONFIG_PATH"),
  155.     # this is used for configuring the default logging level
  156.     "VLLM_LOGGING_LEVEL":
  157.     lambda: os.getenv("VLLM_LOGGING_LEVEL", "INFO"),
  159.     # Trace function calls
  160.     # If set to 1, vllm will trace function calls
  161.     # Useful for debugging
  162.     "VLLM_TRACE_FUNCTION":
  163.     lambda: int(os.getenv("VLLM_TRACE_FUNCTION", "0")),
  165.     # Backend for attention computation
  166.     # Available options:
  167.     # - "TORCH_SDPA": use torch.nn.MultiheadAttention
  168.     # - "FLASH_ATTN": use FlashAttention
  169.     # - "XFORMERS": use XFormers
  170.     # - "ROCM_FLASH": use ROCmFlashAttention
  171.     # - "FLASHINFER": use flashinfer
  172.     "VLLM_ATTENTION_BACKEND":
  173.     lambda: os.getenv("VLLM_ATTENTION_BACKEND", None),
  175.     # CPU key-value cache space
  176.     # default is 4GB
  177.     "VLLM_CPU_KVCACHE_SPACE":
  178.     lambda: int(os.getenv("VLLM_CPU_KVCACHE_SPACE", "0")),
  180.     # OpenVINO key-value cache space
  181.     # default is 4GB
  182.     "VLLM_OPENVINO_KVCACHE_SPACE":
  183.     lambda: int(os.getenv("VLLM_OPENVINO_KVCACHE_SPACE", "0")),
  185.     # OpenVINO KV cache precision
  186.     # default is bf16 if natively supported by platform, otherwise f16
  187.     # To enable KV cache compression, please, explicitly specify u8
  188.     "VLLM_OPENVINO_CPU_KV_CACHE_PRECISION":
  189.     lambda: os.getenv("VLLM_OPENVINO_CPU_KV_CACHE_PRECISION", None),
  191.     # Enables weights compression during model export via HF Optimum
  192.     # default is False
  193.     "VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS":
  194.     lambda: bool(os.getenv("VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS", False)),
  196.     # If the env var is set, then all workers will execute as separate
  197.     # processes from the engine, and we use the same mechanism to trigger
  198.     # execution on all workers.
  199.     # Run vLLM with VLLM_USE_RAY_SPMD_WORKER=1 to enable it.
  200.     "VLLM_USE_RAY_SPMD_WORKER":
  201.     lambda: bool(os.getenv("VLLM_USE_RAY_SPMD_WORKER", 0)),
  203.     # If the env var is set, it uses the Ray's compiled DAG API
  204.     # which optimizes the control plane overhead.
  205.     # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
  206.     "VLLM_USE_RAY_COMPILED_DAG":
  207.     lambda: bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0)),
  209.     # Use dedicated multiprocess context for workers.
  210.     # Both spawn and fork work
  211.     "VLLM_WORKER_MULTIPROC_METHOD":
  212.     lambda: os.getenv("VLLM_WORKER_MULTIPROC_METHOD", "fork"),
  214.     # Path to the cache for storing downloaded assets
  215.     "VLLM_ASSETS_CACHE":
  216.     lambda: os.path.expanduser(
  217.         os.getenv(
  218.             "VLLM_ASSETS_CACHE",
  219.             os.path.join(get_default_cache_root(), "vllm", "assets"),
  220.         )),
  222.     # Timeout for fetching images when serving multimodal models
  223.     # Default is 5 seconds
  224.     "VLLM_IMAGE_FETCH_TIMEOUT":
  225.     lambda: int(os.getenv("VLLM_IMAGE_FETCH_TIMEOUT", "5")),
  227.     # Path to the XLA persistent cache directory.
  228.     # Only used for XLA devices such as TPUs.
  229.     "VLLM_XLA_CACHE_PATH":
  230.     lambda: os.path.expanduser(
  231.         os.getenv(
  232.             "VLLM_ASSETS_CACHE",
  233.             os.path.join(get_default_cache_root(), "vllm", "xla_cache"),
  234.         )),
  235.     "VLLM_FUSED_MOE_CHUNK_SIZE":
  236.     lambda: int(os.getenv("VLLM_FUSED_MOE_CHUNK_SIZE", "65536")),
  238.     # If set, vllm will skip the deprecation warnings.
  239.     "VLLM_NO_DEPRECATION_WARNING":
  240.     lambda: bool(int(os.getenv("VLLM_NO_DEPRECATION_WARNING", "0"))),
  241. }