From c5e5d80361515bd251301077b5b51cc36fdc04a7 Mon Sep 17 00:00:00 2001 From: Adrien Mannocci Date: Wed, 21 Oct 2026 11:28:29 +0200 Subject: [PATCH] feat: default --group-concurrency to the usable CPU core count With `--strategy parallel` and no `--group-concurrency`, each wave used ThreadPoolExecutor's default of min(32, cpu + 4) workers. Each worker starts a terraform/tofu process, so this oversubscribed the CPUs. The default is now `os.process_cpu_count()`, which honors CPU affinity. The `--group-concurrency` option is also defined once in `commands/helpers.py` and shared by plan, apply, fmt and validate. --- README.md | 2 +- src/terranova/commands/apply.py | 10 ++-------- src/terranova/commands/fmt.py | 4 ++-- src/terranova/commands/helpers.py | 7 ++++--- src/terranova/commands/plan.py | 11 ++--------- src/terranova/commands/validate.py | 4 ++-- src/terranova/executor.py | 16 +++++++++++++++- 7 files changed, 28 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 27d44f2..412b6d7 100644 --- a/README.md +++ b/README.md @@ -319,7 +319,7 @@ engine: - `plan`, `apply`, `fmt` and `validate` also accept `--strategy parallel` to run independent resource groups concurrently instead. - Use `--group-concurrency ` to cap how many resource groups run at once under `--strategy parallel` - (defaults to a sane pool size if unset). This is distinct from `--parallelism`, which limits terraform's + (defaults to the number of CPU cores if unset). This is distinct from `--parallelism`, which limits terraform's own resource-level concurrency inside a single invocation. - Use `--fail-at-end` to let unaffected resource groups keep running after a failure instead of stopping the whole run immediately. diff --git a/src/terranova/commands/apply.py b/src/terranova/commands/apply.py index 09deca6..91dbde6 100644 --- a/src/terranova/commands/apply.py +++ b/src/terranova/commands/apply.py @@ -31,6 +31,7 @@ complete_resource_path, execute_tasks, expand_with_dependents, + group_concurrency_option, include_dependents_option, parse_execution_plan, read_manifests_and_waves, @@ -125,14 +126,7 @@ def run(self) -> None: type=click.Choice(["sequential", "parallel"], case_sensitive=False), default="sequential", ) -@click.option( - "--group-concurrency", - help="With `--strategy parallel`, the maximum number of resource groups run " - + "concurrently within a wave. Distinct from terraform's own per-process " - + "resource concurrency. Defaults to a sane pool size if unset.", - type=int, - default=None, -) +@group_concurrency_option @click.pass_obj def apply( ctx: AppContext, diff --git a/src/terranova/commands/fmt.py b/src/terranova/commands/fmt.py index e4800f7..1f68825 100644 --- a/src/terranova/commands/fmt.py +++ b/src/terranova/commands/fmt.py @@ -23,9 +23,9 @@ TerraformTask, complete_resource_path, execute_tasks, - flat_group_concurrency_option, flat_strategy_option, flat_wave, + group_concurrency_option, resource_dirs, ) from terranova.executor import ResourceGroupTask @@ -47,7 +47,7 @@ def run(self) -> None: @click.command("fmt") @click.argument("path", type=str, required=False, shell_complete=complete_resource_path) @flat_strategy_option -@flat_group_concurrency_option +@group_concurrency_option @click.pass_obj def fmt( ctx: AppContext, path: str | None, strategy: str, group_concurrency: int | None diff --git a/src/terranova/commands/helpers.py b/src/terranova/commands/helpers.py index 95c2f61..b93afab 100644 --- a/src/terranova/commands/helpers.py +++ b/src/terranova/commands/helpers.py @@ -89,14 +89,15 @@ ordering concern (`fmt`, `validate` - see `terranova.commands.execution.flat_wave`). `plan`/`apply` define their own, worded around dependency order.""" -flat_group_concurrency_option = click.option( +group_concurrency_option = click.option( "--group-concurrency", help="With `--strategy parallel`, the maximum number of resource groups run " - + "concurrently. Defaults to a sane pool size if unset.", + + "concurrently. Distinct from terraform's own per-process resource " + + "concurrency. Defaults to the number of CPU cores.", type=int, default=None, ) -"""Shared `--group-concurrency` option, paired with `flat_strategy_option`.""" +"""Shared `--group-concurrency` option, used by `plan`, `apply`, `fmt` and `validate`.""" class SelectorType(click.ParamType[Selector]): diff --git a/src/terranova/commands/plan.py b/src/terranova/commands/plan.py index 26f68df..ec2981c 100644 --- a/src/terranova/commands/plan.py +++ b/src/terranova/commands/plan.py @@ -30,6 +30,7 @@ complete_resource_path, execute_tasks, expand_with_dependents, + group_concurrency_option, include_dependents_option, read_manifests_and_waves, resolve_resource_dirs, @@ -164,15 +165,7 @@ def run(self) -> None: type=click.Choice(["sequential", "parallel"], case_sensitive=False), default="sequential", ) -@click.option( - "--group-concurrency", - help="With `--strategy parallel`, the maximum number of resource groups run " - + "concurrently within a wave. Distinct from `--parallelism`, which limits " - + "resource-level concurrency inside a single terraform invocation. " - + "Defaults to a sane pool size if unset.", - type=int, - default=None, -) +@group_concurrency_option @click.pass_obj def plan( ctx: AppContext, diff --git a/src/terranova/commands/validate.py b/src/terranova/commands/validate.py index bd62cf0..b3a75d8 100644 --- a/src/terranova/commands/validate.py +++ b/src/terranova/commands/validate.py @@ -23,9 +23,9 @@ complete_resource_path, discover_resources, execute_tasks, - flat_group_concurrency_option, flat_strategy_option, flat_wave, + group_concurrency_option, resource_dirs, ) from terranova.exceptions import InvalidResourcesError @@ -74,7 +74,7 @@ def run(self) -> None: is_flag=True, ) @flat_strategy_option -@flat_group_concurrency_option +@group_concurrency_option @click.pass_obj def validate( ctx: AppContext, diff --git a/src/terranova/executor.py b/src/terranova/executor.py index 076831b..792f609 100644 --- a/src/terranova/executor.py +++ b/src/terranova/executor.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os from abc import ABC, abstractmethod from concurrent.futures import Future, ThreadPoolExecutor, as_completed from dataclasses import dataclass @@ -24,6 +25,17 @@ from terranova.process import ErrorReturnCode +def default_group_concurrency() -> int: + """ + Default cap on resource groups run concurrently under `--strategy parallel`. + + Each group runs a terraform/tofu process, which is CPU-heavy, so one slot + per usable core (honoring CPU affinity) is a sensible ceiling. Falls back + to 1 when the core count is unknown. + """ + return os.process_cpu_count() or 1 + + class ResourceGroupTask(ABC): """One resource group's unit of work for an executor. Subclasses implement `run()`.""" @@ -164,7 +176,9 @@ def __init__( observer: ExecutorObserver | None = None, ) -> None: """Init the parallel executor.""" - self.__max_workers = max_workers + self.__max_workers = ( + max_workers if max_workers is not None else default_group_concurrency() + ) self.__observer: ExecutorObserver = ( observer if observer is not None else _NoopObserver() )