From b7955ef17b8d899327b25564f20665ec3ffa71cb Mon Sep 17 00:00:00 2001 From: Zhuohan Li Date: Fri, 19 May 2023 14:00:46 -0600 Subject: [PATCH] Fix timeout error in the FastAPI frontend (#34) --- cacheflow/frontend/fastapi_frontend.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cacheflow/frontend/fastapi_frontend.py b/cacheflow/frontend/fastapi_frontend.py index cb7fcc2b17c2..76cb062256b0 100644 --- a/cacheflow/frontend/fastapi_frontend.py +++ b/cacheflow/frontend/fastapi_frontend.py @@ -83,7 +83,7 @@ class FastAPIServer: self.is_server_running = True updated_seq_groups = await self.server.step.remote() self.is_server_running = False - # Notify the waiting coroutines that there new outputs ready. + # Notify the waiting coroutines that there are new outputs ready. for seq_group in updated_seq_groups: group_id = seq_group.group_id self.running_seq_groups[group_id] = seq_group @@ -121,7 +121,10 @@ class FastAPIServer: # Wait for new output. The group_event will be set in server_step # when there is new output available for the sequence group. # Added a timeout to prevent deadlock. - await asyncio.wait_for(group_event.wait(), timeout=TIMEOUT_TO_PREVENT_DEADLOCK) + try: + await asyncio.wait_for(group_event.wait(), timeout=TIMEOUT_TO_PREVENT_DEADLOCK) + except asyncio.TimeoutError: + continue # Reset the event to wait for the next output. group_event.clear() # Decode and return new outputs