Environment Variables

vLLM uses the following environment variables to configure the system:

Warning

Please note that VLLM_PORT and VLLM_HOST_IP set the port and ip for vLLM’s internal usage. It is not the port and ip for the API server. If you use --host $VLLM_HOST_IP and --port $VLLM_PORT to start the API server, it will not work.

All environment variables used by vLLM are prefixed with VLLM_. Special care should be taken for Kubernetes users: please do not name the service as vllm, otherwise environment variables set by Kubernetes might conflict with vLLM’s environment variables, because Kubernetes sets environment variables for each service with the capitalized service name as the prefix.

  2. environment_variables: Dict[str, Callable[[], Any]] = {
  4.     # ================== Installation Time Env Vars ==================
  6.     # Target device of vLLM, supporting [cuda (by default),
  7.     # rocm, neuron, cpu, openvino]
  8.     "VLLM_TARGET_DEVICE":
  9.     lambda: os.getenv("VLLM_TARGET_DEVICE", "cuda"),
  11.     # Maximum number of compilation jobs to run in parallel.
  12.     # By default this is the number of CPUs
  13.     "MAX_JOBS":
  14.     lambda: os.getenv("MAX_JOBS", None),
  16.     # Number of threads to use for nvcc
  17.     # By default this is 1.
  18.     # If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU.
  19.     "NVCC_THREADS":
  20.     lambda: os.getenv("NVCC_THREADS", None),
  22.     # If set, vllm will use precompiled binaries (*.so)
  23.     "VLLM_USE_PRECOMPILED":
  24.     lambda: bool(os.environ.get("VLLM_USE_PRECOMPILED")),
  26.     # CMake build type
  27.     # If not set, defaults to "Debug" or "RelWithDebInfo"
  28.     # Available options: "Debug", "Release", "RelWithDebInfo"
  29.     "CMAKE_BUILD_TYPE":
  30.     lambda: os.getenv("CMAKE_BUILD_TYPE"),
  32.     # If set, vllm will print verbose logs during installation
  33.     "VERBOSE":
  34.     lambda: bool(int(os.getenv('VERBOSE', '0'))),
  36.     # Root directory for VLLM configuration files
  37.     # Defaults to `~/.config/vllm` unless `XDG_CONFIG_HOME` is set
  38.     # Note that this not only affects how vllm finds its configuration files
  39.     # during runtime, but also affects how vllm installs its configuration
  40.     # files during **installation**.
  41.     "VLLM_CONFIG_ROOT":
  42.     lambda: os.path.expanduser(
  43.         os.getenv(
  44.             "VLLM_CONFIG_ROOT",
  45.             os.path.join(get_default_config_root(), "vllm"),
  46.         )),
  48.     # ================== Runtime Env Vars ==================
  50.     # Root directory for VLLM cache files
  51.     # Defaults to `~/.cache/vllm` unless `XDG_CACHE_HOME` is set
  52.     "VLLM_CACHE_ROOT":
  53.     lambda: os.path.expanduser(
  54.         os.getenv(
  55.             "VLLM_CACHE_ROOT",
  56.             os.path.join(get_default_cache_root(), "vllm"),
  57.         )),
  59.     # used in distributed environment to determine the ip address
  60.     # of the current node, when the node has multiple network interfaces.
  61.     # If you are using multi-node inference, you should set this differently
  62.     # on each node.
  63.     'VLLM_HOST_IP':
  64.     lambda: os.getenv('VLLM_HOST_IP', "") or os.getenv("HOST_IP", ""),
  66.     # used in distributed environment to manually set the communication port
  67.     # Note: if VLLM_PORT is set, and some code asks for multiple ports, the
  68.     # VLLM_PORT will be used as the first port, and the rest will be generated
  69.     # by incrementing the VLLM_PORT value.
  70.     # '0' is used to make mypy happy
  71.     'VLLM_PORT':
  72.     lambda: int(os.getenv('VLLM_PORT', '0'))
  73.     if 'VLLM_PORT' in os.environ else None,
  75.     # path used for ipc when the frontend api server is running in
  76.     # multi-processing mode to communicate with the backend engine process.
  77.     'VLLM_RPC_BASE_PATH':
  78.     lambda: os.getenv('VLLM_RPC_BASE_PATH', tempfile.gettempdir()),
  80.     # If true, will load models from ModelScope instead of Hugging Face Hub.
  81.     # note that the value is true or false, not numbers
  82.     "VLLM_USE_MODELSCOPE":
  83.     lambda: os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true",
  85.     # Instance id represents an instance of the VLLM. All processes in the same
  86.     # instance should have the same instance id.
  87.     "VLLM_INSTANCE_ID":
  88.     lambda: os.environ.get("VLLM_INSTANCE_ID", None),
  90.     # Interval in seconds to log a warning message when the ring buffer is full
  91.     "VLLM_RINGBUFFER_WARNING_INTERVAL":
  92.     lambda: int(os.environ.get("VLLM_RINGBUFFER_WARNING_INTERVAL", "60")),
  94.     # path to cudatoolkit home directory, under which should be bin, include,
  95.     # and lib directories.
  96.     "CUDA_HOME":
  97.     lambda: os.environ.get("CUDA_HOME", None),
  99.     # Path to the NCCL library file. It is needed because nccl>=2.19 brought
  100.     # by PyTorch contains a bug: https://github.com/NVIDIA/nccl/issues/1234
  101.     "VLLM_NCCL_SO_PATH":
  102.     lambda: os.environ.get("VLLM_NCCL_SO_PATH", None),
  104.     # when `VLLM_NCCL_SO_PATH` is not set, vllm will try to find the nccl
  105.     # library file in the locations specified by `LD_LIBRARY_PATH`
  106.     "LD_LIBRARY_PATH":
  107.     lambda: os.environ.get("LD_LIBRARY_PATH", None),
  109.     # flag to control if vllm should use triton flash attention
  110.     "VLLM_USE_TRITON_FLASH_ATTN":
  111.     lambda: (os.environ.get("VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in
  112.              ("true", "1")),
  114.     # Internal flag to enable Dynamo graph capture
  115.     "VLLM_TEST_DYNAMO_GRAPH_CAPTURE":
  116.     lambda: int(os.environ.get("VLLM_TEST_DYNAMO_GRAPH_CAPTURE", "0")),
  118.     # local rank of the process in the distributed setting, used to determine
  119.     # the GPU device id
  120.     "LOCAL_RANK":
  121.     lambda: int(os.environ.get("LOCAL_RANK", "0")),
  123.     # used to control the visible devices in the distributed setting
  124.     "CUDA_VISIBLE_DEVICES":
  125.     lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None),
  127.     # timeout for each iteration in the engine
  128.     "VLLM_ENGINE_ITERATION_TIMEOUT_S":
  129.     lambda: int(os.environ.get("VLLM_ENGINE_ITERATION_TIMEOUT_S", "60")),
  131.     # API key for VLLM API server
  132.     "VLLM_API_KEY":
  133.     lambda: os.environ.get("VLLM_API_KEY", None),
  135.     # S3 access information, used for tensorizer to load model from S3
  136.     "S3_ACCESS_KEY_ID":
  137.     lambda: os.environ.get("S3_ACCESS_KEY_ID", None),
  138.     "S3_SECRET_ACCESS_KEY":
  139.     lambda: os.environ.get("S3_SECRET_ACCESS_KEY", None),
  140.     "S3_ENDPOINT_URL":
  141.     lambda: os.environ.get("S3_ENDPOINT_URL", None),
  143.     # Usage stats collection
  144.     "VLLM_USAGE_STATS_SERVER":
  145.     lambda: os.environ.get("VLLM_USAGE_STATS_SERVER", "https://stats.vllm.ai"),
  146.     "VLLM_NO_USAGE_STATS":
  147.     lambda: os.environ.get("VLLM_NO_USAGE_STATS", "0") == "1",
  148.     "VLLM_DO_NOT_TRACK":
  149.     lambda: (os.environ.get("VLLM_DO_NOT_TRACK", None) or os.environ.get(
  150.         "DO_NOT_TRACK", None) or "0") == "1",
  151.     "VLLM_USAGE_SOURCE":
  152.     lambda: os.environ.get("VLLM_USAGE_SOURCE", "production"),
  154.     # Logging configuration
  155.     # If set to 0, vllm will not configure logging
  156.     # If set to 1, vllm will configure logging using the default configuration
  157.     #    or the configuration file specified by VLLM_LOGGING_CONFIG_PATH
  158.     "VLLM_CONFIGURE_LOGGING":
  159.     lambda: int(os.getenv("VLLM_CONFIGURE_LOGGING", "1")),
  160.     "VLLM_LOGGING_CONFIG_PATH":
  161.     lambda: os.getenv("VLLM_LOGGING_CONFIG_PATH"),
  163.     # this is used for configuring the default logging level
  164.     "VLLM_LOGGING_LEVEL":
  165.     lambda: os.getenv("VLLM_LOGGING_LEVEL", "INFO"),
  167.     # Trace function calls
  168.     # If set to 1, vllm will trace function calls
  169.     # Useful for debugging
  170.     "VLLM_TRACE_FUNCTION":
  171.     lambda: int(os.getenv("VLLM_TRACE_FUNCTION", "0")),
  173.     # Backend for attention computation
  174.     # Available options:
  175.     # - "TORCH_SDPA": use torch.nn.MultiheadAttention
  176.     # - "FLASH_ATTN": use FlashAttention
  177.     # - "XFORMERS": use XFormers
  178.     # - "ROCM_FLASH": use ROCmFlashAttention
  179.     # - "FLASHINFER": use flashinfer
  180.     "VLLM_ATTENTION_BACKEND":
  181.     lambda: os.getenv("VLLM_ATTENTION_BACKEND", None),
  183.     # If set, vllm will use flashinfer sampler
  184.     "VLLM_USE_FLASHINFER_SAMPLER":
  185.     lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_SAMPLER", "0"))),
  187.     # Pipeline stage partition strategy
  188.     "VLLM_PP_LAYER_PARTITION":
  189.     lambda: os.getenv("VLLM_PP_LAYER_PARTITION", None),
  191.     # (CPU backend only) CPU key-value cache space.
  192.     # default is 4GB
  193.     "VLLM_CPU_KVCACHE_SPACE":
  194.     lambda: int(os.getenv("VLLM_CPU_KVCACHE_SPACE", "0")),
  196.     # (CPU backend only) CPU core ids bound by OpenMP threads, e.g., "0-31",
  197.     # "0,1,2", "0-31,33". CPU cores of different ranks are separated by '|'.
  198.     "VLLM_CPU_OMP_THREADS_BIND":
  199.     lambda: os.getenv("VLLM_CPU_OMP_THREADS_BIND", "all"),
  201.     # OpenVINO key-value cache space
  202.     # default is 4GB
  203.     "VLLM_OPENVINO_KVCACHE_SPACE":
  204.     lambda: int(os.getenv("VLLM_OPENVINO_KVCACHE_SPACE", "0")),
  206.     # OpenVINO KV cache precision
  207.     # default is bf16 if natively supported by platform, otherwise f16
  208.     # To enable KV cache compression, please, explicitly specify u8
  209.     "VLLM_OPENVINO_CPU_KV_CACHE_PRECISION":
  210.     lambda: os.getenv("VLLM_OPENVINO_CPU_KV_CACHE_PRECISION", None),
  212.     # Enables weights compression during model export via HF Optimum
  213.     # default is False
  214.     "VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS":
  215.     lambda: bool(os.getenv("VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS", False)),
  217.     # If the env var is set, then all workers will execute as separate
  218.     # processes from the engine, and we use the same mechanism to trigger
  219.     # execution on all workers.
  220.     # Run vLLM with VLLM_USE_RAY_SPMD_WORKER=1 to enable it.
  221.     "VLLM_USE_RAY_SPMD_WORKER":
  222.     lambda: bool(int(os.getenv("VLLM_USE_RAY_SPMD_WORKER", "0"))),
  224.     # If the env var is set, it uses the Ray's compiled DAG API
  225.     # which optimizes the control plane overhead.
  226.     # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
  227.     "VLLM_USE_RAY_COMPILED_DAG":
  228.     lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG", "0"))),
  230.     # If the env var is set, it uses NCCL for communication in
  231.     # Ray's compiled DAG. This flag is ignored if
  232.     # VLLM_USE_RAY_COMPILED_DAG is not set.
  233.     "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL":
  234.     lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL", "1"))
  235.                  ),
  237.     # Use dedicated multiprocess context for workers.
  238.     # Both spawn and fork work
  239.     "VLLM_WORKER_MULTIPROC_METHOD":
  240.     lambda: os.getenv("VLLM_WORKER_MULTIPROC_METHOD", "fork"),
  242.     # Path to the cache for storing downloaded assets
  243.     "VLLM_ASSETS_CACHE":
  244.     lambda: os.path.expanduser(
  245.         os.getenv(
  246.             "VLLM_ASSETS_CACHE",
  247.             os.path.join(get_default_cache_root(), "vllm", "assets"),
  248.         )),
  250.     # Timeout for fetching images when serving multimodal models
  251.     # Default is 5 seconds
  252.     "VLLM_IMAGE_FETCH_TIMEOUT":
  253.     lambda: int(os.getenv("VLLM_IMAGE_FETCH_TIMEOUT", "5")),
  255.     # Timeout for fetching audio when serving multimodal models
  256.     # Default is 5 seconds
  257.     "VLLM_AUDIO_FETCH_TIMEOUT":
  258.     lambda: int(os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "5")),
  260.     # Path to the XLA persistent cache directory.
  261.     # Only used for XLA devices such as TPUs.
  262.     "VLLM_XLA_CACHE_PATH":
  263.     lambda: os.path.expanduser(
  264.         os.getenv(
  265.             "VLLM_XLA_CACHE_PATH",
  266.             os.path.join(get_default_cache_root(), "vllm", "xla_cache"),
  267.         )),
  268.     "VLLM_FUSED_MOE_CHUNK_SIZE":
  269.     lambda: int(os.getenv("VLLM_FUSED_MOE_CHUNK_SIZE", "65536")),
  271.     # If set, vllm will skip the deprecation warnings.
  272.     "VLLM_NO_DEPRECATION_WARNING":
  273.     lambda: bool(int(os.getenv("VLLM_NO_DEPRECATION_WARNING", "0"))),
  275.     # If set, the OpenAI API server will stay alive even after the underlying
  276.     # AsyncLLMEngine errors and stops serving requests
  277.     "VLLM_KEEP_ALIVE_ON_ENGINE_DEATH":
  278.     lambda: bool(os.getenv("VLLM_KEEP_ALIVE_ON_ENGINE_DEATH", 0)),
  280.     # If the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN is set, it allows
  281.     # the user to specify a max sequence length greater than
  282.     # the max length derived from the model's config.json.
  283.     # To enable this, set VLLM_ALLOW_LONG_MAX_MODEL_LEN=1.
  284.     "VLLM_ALLOW_LONG_MAX_MODEL_LEN":
  285.     lambda:
  286.     (os.environ.get("VLLM_ALLOW_LONG_MAX_MODEL_LEN", "0").strip().lower() in
  287.      ("1", "true")),
  289.     # If set, forces FP8 Marlin to be used for FP8 quantization regardless
  290.     # of the hardware support for FP8 compute.
  291.     "VLLM_TEST_FORCE_FP8_MARLIN":
  292.     lambda:
  293.     (os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower() in
  294.      ("1", "true")),
  296.     # Time in ms for the zmq client to wait for a response from the backend
  297.     # server for simple data operations
  298.     "VLLM_RPC_GET_DATA_TIMEOUT_MS":
  299.     lambda: int(os.getenv("VLLM_RPC_GET_DATA_TIMEOUT_MS", "5000")),
  301.     # If set, allow running the engine as a separate ray actor,
  302.     # which is a deprecated feature soon to be removed.
  303.     # See https://github.com/vllm-project/vllm/issues/7045
  304.     "VLLM_ALLOW_ENGINE_USE_RAY":
  305.     lambda:
  306.     (os.environ.get("VLLM_ALLOW_ENGINE_USE_RAY", "0").strip().lower() in
  307.      ("1", "true")),
  309.     # a list of plugin names to load, separated by commas.
  310.     # if this is not set, it means all plugins will be loaded
  311.     # if this is set to an empty string, no plugins will be loaded
  312.     "VLLM_PLUGINS":
  313.     lambda: None if "VLLM_PLUGINS" not in os.environ else os.environ[
  314.         "VLLM_PLUGINS"].split(","),
  316.     # Enables torch profiler if set. Path to the directory where torch profiler
  317.     # traces are saved. Note that it must be an absolute path.
  318.     "VLLM_TORCH_PROFILER_DIR":
  319.     lambda: (None if os.getenv("VLLM_TORCH_PROFILER_DIR", None) is None else os
  320.              .path.expanduser(os.getenv("VLLM_TORCH_PROFILER_DIR", "."))),
  321. }