diff --git a/examples/test/run.py b/examples/test/run.py index 6e5934b..097800a 100644 --- a/examples/test/run.py +++ b/examples/test/run.py @@ -1,27 +1,38 @@ import os, shutil, multiprocessing, time -def run_inference(script_path): - output_path = os.path.join("data", script_path) - for script in os.listdir(script_path): - if not script.endswith(".py"): +def script_is_processed(output_path, script): + return os.path.exists(os.path.join(output_path, script)) + + +def filter_unprocessed_tasks(script_path, output_path): + tasks = [] + for script in sorted(os.listdir(script_path)): + if not script.endswith(".sh") and not script.endswith(".py"): continue + if os.path.exists(os.path.join(output_path, script)): + continue + tasks.append(script) + return tasks + + +def run_inference(script_path, tasks): + output_path = os.path.join("data", script_path) + for script in tasks: source_path = os.path.join(script_path, script) target_path = os.path.join(output_path, script) os.makedirs(target_path, exist_ok=True) cmd = f"python {source_path} > {target_path}/log.txt 2>&1" - print(cmd) + print(cmd, flush=True) os.system(cmd) for file_name in os.listdir("./"): if file_name.endswith(".jpg") or file_name.endswith(".png") or file_name.endswith(".mp4"): shutil.move(file_name, os.path.join(target_path, file_name)) -def run_tasks_on_single_GPU(script_path, gpu_id, num_gpu): +def run_tasks_on_single_GPU(script_path, tasks, gpu_id, num_gpu): output_path = os.path.join("data", script_path) - for script_id, script in enumerate(sorted(os.listdir(script_path))): - if not script.endswith(".sh") and not script.endswith(".py"): - continue + for script_id, script in enumerate(tasks): if script_id % num_gpu != gpu_id: continue source_path = os.path.join(script_path, script) @@ -35,11 +46,9 @@ def run_tasks_on_single_GPU(script_path, gpu_id, num_gpu): os.system(cmd) -def run_train_multi_GPU(script_path): +def run_train_multi_GPU(script_path, tasks): output_path = os.path.join("data", script_path) - for script in os.listdir(script_path)[::-1]: - if not script.endswith(".sh"): - continue + for script in tasks: source_path = os.path.join(script_path, script) target_path = os.path.join(output_path, script) os.makedirs(target_path, exist_ok=True) @@ -79,5 +88,6 @@ if __name__ == "__main__": # run_train_single_GPU("examples/wanvideo/model_training/validate_lora") # move_files("video_", "data/output/validate_lora") # run_train_multi_GPU("examples/wanvideo/model_training/full") - run_train_single_GPU("examples/wanvideo/model_training/validate_full") - move_files("video_", "data/output/validate_full") + # run_train_single_GPU("examples/wanvideo/model_training/validate_full") + # move_files("video_", "data/output/validate_full") + pass