theluyuan/RWKV-Runner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

improve api concurrency performance

Browse Source
This commit is contained in:
josc146 2023-05-27 15:18:12 +08:00
parent 2ca8f5eba9
commit b7fb8ed898
1 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
backend-python/routes/completion.py
View File

@ -93,6 +93,8 @@ The following is a coherent verbose detailed conversation between a girl named {
async def eval_rwkv(): async def eval_rwkv():
while completion_lock.locked(): while completion_lock.locked():
if await request.is_disconnected():
return
await asyncio.sleep(0.1) await asyncio.sleep(0.1)
else: else:
completion_lock.acquire() completion_lock.acquire()
@ -185,8 +187,13 @@ async def completions(body: CompletionBody, request: Request):
if model is None: if model is None:
raise HTTPException(status.HTTP_400_BAD_REQUEST, "model not loaded") raise HTTPException(status.HTTP_400_BAD_REQUEST, "model not loaded")
if body.prompt is None or body.prompt == "":
raise HTTPException(status.HTTP_400_BAD_REQUEST, "prompt not found")
async def eval_rwkv(): async def eval_rwkv():
while completion_lock.locked(): while completion_lock.locked():
if await request.is_disconnected():
return
await asyncio.sleep(0.1) await asyncio.sleep(0.1)
else: else:
completion_lock.acquire() completion_lock.acquire()