xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-10 09:35:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Set enable_prefix_caching=True in prefix caching example (#3703)

Browse Source
This commit is contained in:
Woosuk Kwon 2024-03-28 16:26:30 -07:00 committed by GitHub
parent cb40b3ab6b
commit c0935c96d3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/offline_inference_with_prefix.py
View File

@ -22,7 +22,7 @@ prompts = [
sampling_params = SamplingParams(temperature=0.0)
# Create an LLM.
llm = LLM(model="facebook/opt-125m")
llm = LLM(model="facebook/opt-125m", enable_prefix_caching=True)
generating_prompts = [prefix + prompt for prompt in prompts]