Offline Inference Chat

Source vllm-project/vllm.

  1.  1from vllm import LLM, SamplingParams
  2.  2
  3.  3llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
  4.  4sampling_params = SamplingParams(temperature=0.5)
  5.  5
  6.  6
  7.  7def print_outputs(outputs):
  8.  8    for output in outputs:
  9.  9        prompt = output.prompt
  10. 10        generated_text = output.outputs[0].text
  11. 11        print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
  12. 12    print("-" * 80)
  13. 13
  14. 14
  15. 15print("=" * 80)
  16. 16
  17. 17# In this script, we demonstrate how to pass input to the chat method:
  18. 18
  19. 19conversation = [
  20. 20    {
  21. 21        "role": "system",
  22. 22        "content": "You are a helpful assistant"
  23. 23    },
  24. 24    {
  25. 25        "role": "user",
  26. 26        "content": "Hello"
  27. 27    },
  28. 28    {
  29. 29        "role": "assistant",
  30. 30        "content": "Hello! How can I assist you today?"
  31. 31    },
  32. 32    {
  33. 33        "role": "user",
  34. 34        "content": "Write an essay about the importance of higher education.",
  35. 35    },
  36. 36]
  37. 37outputs = llm.chat(conversation,
  38. 38                   sampling_params=sampling_params,
  39. 39                   use_tqdm=False)
  40. 40print_outputs(outputs)
  41. 41
  42. 42# A chat template can be optionally supplied.
  43. 43# If not, the model will use its default chat template.
  44. 44
  45. 45# with open('template_falcon_180b.jinja', "r") as f:
  46. 46#     chat_template = f.read()
  47. 47
  48. 48# outputs = llm.chat(
  49. 49#     conversations,
  50. 50#     sampling_params=sampling_params,
  51. 51#     use_tqdm=False,
  52. 52#     chat_template=chat_template,
  53. 53# )