improve state cache performance
This commit is contained in:
@@ -239,14 +239,9 @@ class AbstractRWKV(ABC):
|
||||
self.model_tokens = []
|
||||
else:
|
||||
delta_prompt = prompt[len(cache["prompt"]) :]
|
||||
state = cache["state"]
|
||||
self.model_state = (
|
||||
copy.deepcopy(state)
|
||||
if type(state) == list or type(state) == np.ndarray
|
||||
else state
|
||||
)
|
||||
self.model_tokens = copy.deepcopy(cache["tokens"])
|
||||
logits = copy.deepcopy(cache["logits"])
|
||||
self.model_state = cache["state"]
|
||||
self.model_tokens = cache["tokens"]
|
||||
logits = cache["logits"]
|
||||
|
||||
prompt_token_len = 0
|
||||
if delta_prompt != "":
|
||||
|
||||
Reference in New Issue
Block a user