xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-10 02:25:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Benchmarks] Remove invalid option under V1 engine (#17651)

Browse Source 
Signed-off-by: Russell Bryant <rbryant@redhat.com>
This commit is contained in:
Russell Bryant 2025-05-05 16:30:22 -04:00 committed by GitHub
parent aea302be6c
commit d3efde8176
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 3 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
benchmarks/benchmark_serving_structured_output.py
View File

@ -414,7 +414,6 @@ async def benchmark(
ignore_eos: bool, ignore_eos: bool,
max_concurrency: Optional[int], max_concurrency: Optional[int],
structured_output_ratio: float, structured_output_ratio: float,
structured_output_backend: str,
goodput_config_dict: Optional[dict[str, float]] = None, goodput_config_dict: Optional[dict[str, float]] = None,
): ):
if backend in ASYNC_REQUEST_FUNCS: if backend in ASYNC_REQUEST_FUNCS:
@ -426,8 +425,6 @@ async def benchmark(
extra_body = {} extra_body = {}
# Add the schema to the extra_body # Add the schema to the extra_body
extra_body[request.structure_type] = request.schema extra_body[request.structure_type] = request.schema
# Add the specific structured_output_backend
extra_body["guided_decoding_backend"] = structured_output_backend
return extra_body return extra_body
print("Starting initial single prompt test run...") print("Starting initial single prompt test run...")
@ -785,7 +782,6 @@ def main(args: argparse.Namespace):
ignore_eos=args.ignore_eos, ignore_eos=args.ignore_eos,
max_concurrency=args.max_concurrency, max_concurrency=args.max_concurrency,
structured_output_ratio=args.structured_output_ratio, structured_output_ratio=args.structured_output_ratio,
structured_output_backend=args.structured_output_backend,
goodput_config_dict=goodput_config_dict, goodput_config_dict=goodput_config_dict,
)) ))
@ -1000,14 +996,6 @@ if __name__ == "__main__":
type=float, type=float,
default=1.0, default=1.0,
help="Ratio of Structured Outputs requests") help="Ratio of Structured Outputs requests")
parser.add_argument("--structured-output-backend",
type=str,
choices=[
"outlines", "lm-format-enforcer", "xgrammar",
"guidance", "auto"
],
default="auto",
help="Backend to use for structured outputs")
args = parser.parse_args() args = parser.parse_args()
main(args) main(args)

11
benchmarks/run_structured_output_benchmark.sh
View File

@ -9,13 +9,10 @@ BACKEND=${2:-"vllm"}
# Define the dataset to use # Define the dataset to use
DATASET=${3:-"xgrammar_bench"} DATASET=${3:-"xgrammar_bench"}
# Define the guided decoding backend
GUIDED_BACKEND=${4:-"xgrammar"}
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
OUTPUT_DIR=${5:-"$SCRIPT_DIR/structured_output_benchmark_results"} OUTPUT_DIR=${4:-"$SCRIPT_DIR/structured_output_benchmark_results"}
GUIDED_RATIO=${6:-0.5} GUIDED_RATIO=${5:-0.5}
# Create output directory if it doesn't exist # Create output directory if it doesn't exist
mkdir -p "$OUTPUT_DIR" mkdir -p "$OUTPUT_DIR"
@ -27,7 +24,6 @@ QPS_VALUES=(70 60 50 25 20 15 10)
COMMON_PARAMS="--backend $BACKEND \ COMMON_PARAMS="--backend $BACKEND \
--model $MODEL \ --model $MODEL \
--dataset $DATASET \ --dataset $DATASET \
--structured-output-backend $GUIDED_BACKEND \
--structured-output-ratio $GUIDED_RATIO \ --structured-output-ratio $GUIDED_RATIO \
--save-results \ --save-results \
--result-dir $OUTPUT_DIR" --result-dir $OUTPUT_DIR"
@ -35,7 +31,6 @@ COMMON_PARAMS="--backend $BACKEND \
echo "Starting structured output benchmark with model: $MODEL" echo "Starting structured output benchmark with model: $MODEL"
echo "Backend: $BACKEND" echo "Backend: $BACKEND"
echo "Dataset: $DATASET" echo "Dataset: $DATASET"
echo "Structured output backend: $GUIDED_BACKEND"
echo "Results will be saved to: $OUTPUT_DIR" echo "Results will be saved to: $OUTPUT_DIR"
echo "----------------------------------------" echo "----------------------------------------"
@ -48,7 +43,7 @@ for qps in "${QPS_VALUES[@]}"; do
GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")
# Construct filename for this run # Construct filename for this run
FILENAME="${GUIDED_BACKEND}_${BACKEND}_${qps}qps_$(basename $MODEL)_${DATASET}_${GIT_HASH}.json" FILENAME="${BACKEND}_${qps}qps_$(basename $MODEL)_${DATASET}_${GIT_HASH}.json"
# Run the benchmark # Run the benchmark
python "$SCRIPT_DIR/benchmark_serving_structured_output.py" $COMMON_PARAMS \ python "$SCRIPT_DIR/benchmark_serving_structured_output.py" $COMMON_PARAMS \