From 6ed6d15a697c15145a51b12a9c0e5222eaa5a3d3 Mon Sep 17 00:00:00 2001 From: Joe Date: Sun, 1 Mar 2026 16:32:34 -0600 Subject: [PATCH] Refactor CI scripts to improve thread management for Rust builds and tests - Renamed thread variables for clarity, changing `$Threads` to `$BuildThreads` and introducing `$TestThreads` to cap test execution at 8 threads to prevent resource exhaustion. - Updated `cargo` commands in both PowerShell and shell scripts to utilize the new thread management variables, ensuring consistent behavior across CI environments. - Added comments to clarify the rationale behind capping test threads, enhancing maintainability and understanding of the CI process. These changes aim to optimize resource usage during CI runs and improve the reliability of Rust builds and tests. --- scripts/ci.ps1 | 20 ++++++++++++-------- scripts/ci.sh | 21 +++++++++++++-------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/scripts/ci.ps1 b/scripts/ci.ps1 index 32bf74fd..ba2199bb 100644 --- a/scripts/ci.ps1 +++ b/scripts/ci.ps1 @@ -6,7 +6,11 @@ param( ) $RbxDom = -not $NoRbxDom -$Threads = [Environment]::ProcessorCount +$BuildThreads = [Environment]::ProcessorCount +# Cap test threads: two_way_sync tests spawn child processes (atlas serve) +# with file watchers, HTTP servers, and temp dirs. Running too many in +# parallel (e.g. 32) exhausts OS resources and causes spurious failures. +$TestThreads = [Math]::Min([Environment]::ProcessorCount, 8) $ErrorActionPreference = "Continue" $failures = @() @@ -57,9 +61,9 @@ $selene = $LASTEXITCODE Record-Result "Selene" $selene Write-Step 6 "Rust Linting (Clippy) - Auto-fix" -cargo clippy -j $Threads --fix --allow-dirty --allow-staged 2>&1 | Out-Null +cargo clippy -j $BuildThreads --fix --allow-dirty --allow-staged 2>&1 | Out-Null Write-Host "Verifying..." -ForegroundColor Yellow -cargo clippy -j $Threads --all-targets --all-features 2>&1 +cargo clippy -j $BuildThreads --all-targets --all-features 2>&1 $clippy = $LASTEXITCODE Record-Result "Clippy" $clippy @@ -69,7 +73,7 @@ $build = $LASTEXITCODE Record-Result "Build" $build Write-Step 8 "Run ALL Rust Tests" -$testOutput = cargo test --locked --all-features -- --test-threads=$Threads 2>&1 +$testOutput = cargo test --locked --all-features -- --test-threads=$TestThreads 2>&1 $rustTests = $LASTEXITCODE $testOutput | Write-Host Record-Result "Rust Tests" $rustTests @@ -107,9 +111,9 @@ if ($RbxDom) { Write-Step 13 "Rust Linting (Clippy) for rbx-dom - Auto-fix" Push-Location rbx-dom - cargo clippy -j $Threads --fix --allow-dirty --allow-staged 2>&1 | Out-Null + cargo clippy -j $BuildThreads --fix --allow-dirty --allow-staged 2>&1 | Out-Null Write-Host "Verifying..." -ForegroundColor Yellow - cargo clippy -j $Threads --all-targets --all-features -- -D warnings 2>&1 + cargo clippy -j $BuildThreads --all-targets --all-features -- -D warnings 2>&1 $clippyRbxDom = $LASTEXITCODE Pop-Location Record-Result "Clippy (rbx-dom)" $clippyRbxDom @@ -126,9 +130,9 @@ if ($RbxDom) { Write-Step 15 "Run ALL rbx-dom Rust Tests" Push-Location rbx-dom - cargo test --verbose -- --test-threads=$Threads + cargo test --verbose -- --test-threads=$TestThreads $testsRbxDom = $LASTEXITCODE - cargo test --all-features --verbose -- --test-threads=$Threads + cargo test --all-features --verbose -- --test-threads=$TestThreads $testsRbxDomAll = $LASTEXITCODE Pop-Location Record-Result "Tests (rbx-dom)" $testsRbxDom diff --git a/scripts/ci.sh b/scripts/ci.sh index a3673ea3..2518d166 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -11,7 +11,12 @@ for arg in "$@"; do esac done -THREADS=$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || echo 4) +BUILD_THREADS=$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || echo 4) +# Cap test threads: two_way_sync tests spawn child processes (atlas serve) +# with file watchers, HTTP servers, and temp dirs. Running too many in +# parallel (e.g. 32) exhausts OS resources and causes spurious failures. +_cpu_count=$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || echo 4) +TEST_THREADS=$(( _cpu_count < 8 ? _cpu_count : 8 )) failures=() @@ -57,9 +62,9 @@ selene_exit=$? record "Selene" $selene_exit step 6 "Rust Linting (Clippy) - Auto-fix" -cargo clippy -j $THREADS --fix --allow-dirty --allow-staged >/dev/null 2>&1 +cargo clippy -j $BUILD_THREADS --fix --allow-dirty --allow-staged >/dev/null 2>&1 echo "Verifying..." -cargo clippy -j $THREADS --all-targets --all-features 2>&1 +cargo clippy -j $BUILD_THREADS --all-targets --all-features 2>&1 clippy_exit=$? record "Clippy" $clippy_exit @@ -69,7 +74,7 @@ build_exit=$? record "Build" $build_exit step 8 "Run ALL Rust Tests" -cargo test --locked --all-features -- --test-threads=$THREADS 2>&1 +cargo test --locked --all-features -- --test-threads=$TEST_THREADS 2>&1 rust_tests=$? record "Rust Tests" $rust_tests @@ -103,9 +108,9 @@ if [ "$RBX_DOM" = true ]; then record "Selene (rbx_dom_lua)" $selene_rbxdom step 13 "Rust Linting (Clippy) for rbx-dom - Auto-fix" - (cd rbx-dom && cargo clippy -j $THREADS --fix --allow-dirty --allow-staged >/dev/null 2>&1) + (cd rbx-dom && cargo clippy -j $BUILD_THREADS --fix --allow-dirty --allow-staged >/dev/null 2>&1) echo "Verifying..." - (cd rbx-dom && cargo clippy -j $THREADS --all-targets --all-features -- -D warnings 2>&1) + (cd rbx-dom && cargo clippy -j $BUILD_THREADS --all-targets --all-features -- -D warnings 2>&1) clippy_rbxdom=$? record "Clippy (rbx-dom)" $clippy_rbxdom @@ -118,9 +123,9 @@ if [ "$RBX_DOM" = true ]; then record "Build (rbx-dom all-features)" $build_rbxdom_all step 15 "Run ALL rbx-dom Rust Tests" - (cd rbx-dom && cargo test --verbose -- --test-threads=$THREADS) + (cd rbx-dom && cargo test --verbose -- --test-threads=$TEST_THREADS) tests_rbxdom=$? - (cd rbx-dom && cargo test --all-features --verbose -- --test-threads=$THREADS) + (cd rbx-dom && cargo test --all-features --verbose -- --test-threads=$TEST_THREADS) tests_rbxdom_all=$? record "Tests (rbx-dom)" $tests_rbxdom record "Tests (rbx-dom all-features)" $tests_rbxdom_all