diff --git a/benchmarks/benchmark_serving_structured_output.py b/benchmarks/benchmark_serving_structured_output.py index 7c40e39ac810..9084255d2440 100644 --- a/benchmarks/benchmark_serving_structured_output.py +++ b/benchmarks/benchmark_serving_structured_output.py @@ -414,7 +414,6 @@ async def benchmark( ignore_eos: bool, max_concurrency: Optional[int], structured_output_ratio: float, - structured_output_backend: str, goodput_config_dict: Optional[dict[str, float]] = None, ): if backend in ASYNC_REQUEST_FUNCS: @@ -426,8 +425,6 @@ async def benchmark( extra_body = {} # Add the schema to the extra_body extra_body[request.structure_type] = request.schema - # Add the specific structured_output_backend - extra_body["guided_decoding_backend"] = structured_output_backend return extra_body print("Starting initial single prompt test run...") @@ -785,7 +782,6 @@ def main(args: argparse.Namespace): ignore_eos=args.ignore_eos, max_concurrency=args.max_concurrency, structured_output_ratio=args.structured_output_ratio, - structured_output_backend=args.structured_output_backend, goodput_config_dict=goodput_config_dict, )) @@ -1000,14 +996,6 @@ if __name__ == "__main__": type=float, default=1.0, help="Ratio of Structured Outputs requests") - parser.add_argument("--structured-output-backend", - type=str, - choices=[ - "outlines", "lm-format-enforcer", "xgrammar", - "guidance", "auto" - ], - default="auto", - help="Backend to use for structured outputs") args = parser.parse_args() main(args) diff --git a/benchmarks/run_structured_output_benchmark.sh b/benchmarks/run_structured_output_benchmark.sh index 126dfbc24416..53dc7ed70b9c 100755 --- a/benchmarks/run_structured_output_benchmark.sh +++ b/benchmarks/run_structured_output_benchmark.sh @@ -9,13 +9,10 @@ BACKEND=${2:-"vllm"} # Define the dataset to use DATASET=${3:-"xgrammar_bench"} -# Define the guided decoding backend -GUIDED_BACKEND=${4:-"xgrammar"} - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -OUTPUT_DIR=${5:-"$SCRIPT_DIR/structured_output_benchmark_results"} +OUTPUT_DIR=${4:-"$SCRIPT_DIR/structured_output_benchmark_results"} -GUIDED_RATIO=${6:-0.5} +GUIDED_RATIO=${5:-0.5} # Create output directory if it doesn't exist mkdir -p "$OUTPUT_DIR" @@ -27,7 +24,6 @@ QPS_VALUES=(70 60 50 25 20 15 10) COMMON_PARAMS="--backend $BACKEND \ --model $MODEL \ --dataset $DATASET \ - --structured-output-backend $GUIDED_BACKEND \ --structured-output-ratio $GUIDED_RATIO \ --save-results \ --result-dir $OUTPUT_DIR" @@ -35,7 +31,6 @@ COMMON_PARAMS="--backend $BACKEND \ echo "Starting structured output benchmark with model: $MODEL" echo "Backend: $BACKEND" echo "Dataset: $DATASET" -echo "Structured output backend: $GUIDED_BACKEND" echo "Results will be saved to: $OUTPUT_DIR" echo "----------------------------------------" @@ -48,7 +43,7 @@ for qps in "${QPS_VALUES[@]}"; do GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") # Construct filename for this run - FILENAME="${GUIDED_BACKEND}_${BACKEND}_${qps}qps_$(basename $MODEL)_${DATASET}_${GIT_HASH}.json" + FILENAME="${BACKEND}_${qps}qps_$(basename $MODEL)_${DATASET}_${GIT_HASH}.json" # Run the benchmark python "$SCRIPT_DIR/benchmark_serving_structured_output.py" $COMMON_PARAMS \