allow setting tokenChunkSize of WebGPU mode
This commit is contained in:
9
backend-python/rwkv_pip/webgpu/model.py
vendored
9
backend-python/rwkv_pip/webgpu/model.py
vendored
@@ -26,12 +26,19 @@ class RWKV:
|
||||
if s.startswith("layer")
|
||||
)
|
||||
|
||||
chunk_size = (
|
||||
int(s.lstrip("chunk"))
|
||||
for s in strategy.split()
|
||||
for s in s.split(",")
|
||||
if s.startswith("chunk")
|
||||
)
|
||||
|
||||
args = {
|
||||
"file": model_path,
|
||||
"turbo": True,
|
||||
"quant": next(layer, 31) if "i8" in strategy else 0,
|
||||
"quant_nf4": next(layer, 26) if "i4" in strategy else 0,
|
||||
"token_chunk_size": 128,
|
||||
"token_chunk_size": next(chunk_size, 32),
|
||||
"lora": None,
|
||||
}
|
||||
self.model = self.wrp.Model(**args)
|
||||
|
||||
Reference in New Issue
Block a user