feat: add ability to specify params to drop

Some LLMs don't support all params that clients could send. This MR gives the administrator the ability to filter those out

Similar to https://docs.litellm.ai/docs/completion/drop_params

Signed-off-by: Max Wittig <[email protected]>

Author: max-wittig

src/vllm_router/app.py

@@ -197,6 +197,8 @@ def initialize_all(app: FastAPI, args):
     if args.callbacks:
         configure_custom_callbacks(args.callbacks, app)
 
+    app.state.drop_params = parse_comma_separated_args(args.drop_params)
+
     initialize_routing_logic(
         args.routing_logic,
         session_key=args.session_key,

src/vllm_router/parsers/parser.py

@@ -228,6 +228,16 @@ def parse_args():
         help="The request rewriter to use. Default is 'noop' (no rewriting).",
     )
 
+    # Drop params arguments
+    parser.add_argument(
+        "--drop-params",
+        type=str,
+        default=None,
+        help="Comma-separated list of OpenAI parameters to drop from requests. "
+        "This allows dropping unsupported parameters by your LLM provider. "
+        "Example: 'frequency_penalty,logit_bias'",
+    )
+
     # Batch API
     # TODO(gaocegege): Make these batch api related arguments to a separate config.
     parser.add_argument(

src/vllm_router/services/request_service/request.py

@@ -203,6 +203,13 @@ async def route_general_request(
                 status_code=400, detail="Request body is not JSON parsable."
             )
 
+    if hasattr(request.app.state, "drop_params") and request.app.state.drop_params:
+        for param in request.app.state.drop_params:
+            request_json.pop(param, None)
+            logger.debug(f"Dropped param {param} from request")
+        request_body = json.dumps(request_json)
+        update_content_length(request, request_body)
+
     # TODO (ApostaC): merge two awaits into one
     service_discovery = get_service_discovery()
     endpoints = service_discovery.get_endpoint_info()