From ec4327e427439ac98eae732aaef9a7e7ec291bbb Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 3 Mar 2025 17:20:41 +0800 Subject: [PATCH 1/2] add task pool for regeneration --- .../eng/scripts/ci/regenerate.ts | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/packages/http-client-python/eng/scripts/ci/regenerate.ts b/packages/http-client-python/eng/scripts/ci/regenerate.ts index fe1628ff3d4..c20d0392c60 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate.ts @@ -316,6 +316,20 @@ function _getCmdList(spec: string, flags: RegenerateFlags): TspCommand[] { }); } +async function runTaskPool(tasks: Array<() => Promise>, poolLimit: number): Promise { + let currentIndex = 0; + + async function worker() { + while (currentIndex < tasks.length) { + const index = currentIndex++; + await tasks[index](); + } + } + + const workers = new Array(Math.min(poolLimit, tasks.length)).fill(null).map(() => worker()); + await Promise.all(workers); +} + async function regenerate(flags: RegenerateFlagsInput): Promise { if (flags.flavor === undefined) { await regenerate({ flavor: "azure", ...flags }); @@ -331,11 +345,22 @@ async function regenerate(flags: RegenerateFlagsInput): Promise { const cmdList: TspCommand[] = subdirectories.flatMap((subdirectory) => _getCmdList(subdirectory, flagsResolved), ); - const PromiseCommands = cmdList.map((tspCommand) => executeCommand(tspCommand)); - await Promise.all(PromiseCommands); + + // Create tasks as functions for the pool + const tasks: Array<() => Promise> = cmdList.map((tspCommand) => { + return () => executeCommand(tspCommand); + }); + + // Run tasks with a concurrency limit + await runTaskPool(tasks, 30); } } +const start = performance.now(); regenerate(argv.values) - .then(() => console.log("Regeneration successful")) + .then(() => + console.log( + `Regeneration successful, time taken: ${Math.round((performance.now() - start) / 1000)} s`, + ), + ) .catch((error) => console.error(`Regeneration failed: ${error.message}`)); From 9901fdfaf1c18dbe24141c5f384f41444af7d3a4 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 3 Mar 2025 17:26:24 +0800 Subject: [PATCH 2/2] Create python-regenerate-optimize-2025-2-3-9-23-57.md --- .../python-regenerate-optimize-2025-2-3-9-23-57.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .chronus/changes/python-regenerate-optimize-2025-2-3-9-23-57.md diff --git a/.chronus/changes/python-regenerate-optimize-2025-2-3-9-23-57.md b/.chronus/changes/python-regenerate-optimize-2025-2-3-9-23-57.md new file mode 100644 index 00000000000..23a2ffea92a --- /dev/null +++ b/.chronus/changes/python-regenerate-optimize-2025-2-3-9-23-57.md @@ -0,0 +1,8 @@ +--- +# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking +changeKind: internal +packages: + - "@typespec/http-client-python" +--- + +add task pool for regeneration