improve python backend startup speed

backend-python/routes/state_cache.py

@@ -4,8 +4,6 @@ from fastapi import APIRouter, HTTPException, Request, Response, status
 from pydantic import BaseModel
 import gc
 import copy
-import sys
-import torch
 
 router = APIRouter()
 
@@ -73,6 +71,8 @@ def add_state(body: AddStateBody):
     if trie is None:
         raise HTTPException(status.HTTP_400_BAD_REQUEST, "trie not loaded")
 
+    import torch
+
     try:
         id: int = trie.insert(body.prompt)
         device: torch.device = body.state[0].device
@@ -147,6 +147,8 @@ def longest_prefix_state(body: LongestPrefixStateBody, request: Request):
     if trie is None:
         raise HTTPException(status.HTTP_400_BAD_REQUEST, "trie not loaded")
 
+    import torch
+
     id = -1
     try:
         for id, len in trie.prefix(body.prompt):