From b2a77060dcd1946b35118000561bda7712335cc0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Jan 2026 00:06:09 +0000 Subject: [PATCH 1/2] Initial plan From 8188cf8c537f2caacbca536c3076829f774a91b3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Jan 2026 00:11:17 +0000 Subject: [PATCH 2/2] Refactor: Replace unbounded thread spawning with rayon parallel iterators - Add rayon 1.11.0 dependency to pet-conda and pet-homebrew crates - Refactor get_conda_environments() in lib.rs to use par_iter() - Refactor get_conda_environment_paths() in environment_locations.rs to use par_iter() - Refactor get_known_symlinks() in sym_links.rs to use par_iter() This provides controlled parallelism based on CPU count, better resource management, and avoids potential thread exhaustion on systems with hundreds of Python environments. Co-authored-by: karthiknadig <3840081+karthiknadig@users.noreply.github.com> --- Cargo.lock | 53 +++++++++++++++++++ crates/pet-conda/Cargo.toml | 1 + crates/pet-conda/src/environment_locations.rs | 19 +++---- crates/pet-conda/src/lib.rs | 25 ++------- crates/pet-homebrew/Cargo.toml | 1 + crates/pet-homebrew/src/sym_links.rs | 43 +++++++-------- 6 files changed, 85 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3ee3ea80..b5dc8bb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -172,6 +172,31 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crypto-common" version = "0.1.6" @@ -192,6 +217,12 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "encoding_rs" version = "0.8.34" @@ -429,6 +460,7 @@ dependencies = [ "pet-fs", "pet-python-utils", "pet-reporter", + "rayon", "regex", "serde", "serde_json", @@ -497,6 +529,7 @@ dependencies = [ "pet-fs", "pet-python-utils", "pet-virtualenv", + "rayon", "regex", "serde", "serde_json", @@ -768,6 +801,26 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "regex" version = "1.10.5" diff --git a/crates/pet-conda/Cargo.toml b/crates/pet-conda/Cargo.toml index 4f324c48..f47e3c12 100644 --- a/crates/pet-conda/Cargo.toml +++ b/crates/pet-conda/Cargo.toml @@ -19,6 +19,7 @@ regex = "1.10.4" pet-reporter = { path = "../pet-reporter" } env_logger = "0.10.2" yaml-rust2 = "0.8.1" +rayon = "1.11.0" [features] ci = [] diff --git a/crates/pet-conda/src/environment_locations.rs b/crates/pet-conda/src/environment_locations.rs index 529a4427..94ae308b 100644 --- a/crates/pet-conda/src/environment_locations.rs +++ b/crates/pet-conda/src/environment_locations.rs @@ -9,6 +9,7 @@ use crate::{ use log::trace; use pet_fs::path::{expand_path, norm_case}; use pet_python_utils::platform_dirs::Platformdirs; +use rayon::prelude::*; use std::{ env, fs, path::{Path, PathBuf}, @@ -43,19 +44,11 @@ pub fn get_conda_environment_paths( env_paths.dedup(); // For each env, check if we have a conda install directory in them and // & then iterate through the list of envs in the envs directory. - // let env_paths = vec![]; - let mut threads = vec![]; - for path in env_paths.iter().filter(|f| f.exists()) { - let path = path.clone(); - threads.push(thread::spawn(move || get_environments(&path))); - } - - let mut result = vec![]; - for thread in threads { - if let Ok(envs) = thread.join() { - result.extend(envs); - } - } + let mut result: Vec = env_paths + .par_iter() + .filter(|f| f.exists()) + .flat_map(|path| get_environments(path)) + .collect(); result.sort(); result.dedup(); diff --git a/crates/pet-conda/src/lib.rs b/crates/pet-conda/src/lib.rs index 59fe084e..d233f67f 100644 --- a/crates/pet-conda/src/lib.rs +++ b/crates/pet-conda/src/lib.rs @@ -18,6 +18,7 @@ use pet_core::{ Locator, LocatorKind, }; use pet_fs::path::norm_case; +use rayon::prelude::*; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, @@ -371,24 +372,8 @@ fn get_conda_environments( paths: &Vec, manager: &Option, ) -> Vec { - let mut threads = vec![]; - for path in paths { - let path = path.clone(); - let mgr = manager.clone(); - threads.push(thread::spawn(move || { - if let Some(env) = get_conda_environment_info(&path, &mgr) { - vec![env] - } else { - vec![] - } - })); - } - - let mut envs: Vec = vec![]; - for thread in threads { - if let Ok(mut result) = thread.join() { - envs.append(&mut result); - } - } - envs + paths + .par_iter() + .filter_map(|path| get_conda_environment_info(path, manager)) + .collect() } diff --git a/crates/pet-homebrew/Cargo.toml b/crates/pet-homebrew/Cargo.toml index 6b107d4a..1adf70c1 100644 --- a/crates/pet-homebrew/Cargo.toml +++ b/crates/pet-homebrew/Cargo.toml @@ -18,3 +18,4 @@ lazy_static = "1.4.0" pet-core = { path = "../pet-core" } log = "0.4.21" regex = "1.10.4" +rayon = "1.11.0" diff --git a/crates/pet-homebrew/src/sym_links.rs b/crates/pet-homebrew/src/sym_links.rs index 9bea4adb..31b53e77 100644 --- a/crates/pet-homebrew/src/sym_links.rs +++ b/crates/pet-homebrew/src/sym_links.rs @@ -4,6 +4,7 @@ use lazy_static::lazy_static; use pet_fs::path::resolve_symlink; use pet_python_utils::executable::find_executables; +use rayon::prelude::*; use regex::Regex; use std::{ fs, @@ -30,32 +31,26 @@ pub fn get_known_symlinks( // Go through all the exes in all of the above bin directories and verify we have a list of all of them. // They too could be symlinks, e.g. we could have `/opt/homebrew/bin/python3` & also `/opt/homebrew/bin/python` // And possible they are all symlnks to the same exe. - let threads = symlinks - .iter() - .map(|symlink| { - let symlink = symlink.clone(); - let known_symlinks = symlinks.clone(); - std::thread::spawn(move || { - if let Some(bin) = symlink.parent() { - let mut symlinks = vec![]; - for possible_symlink in find_executables(bin) { - if let Some(symlink) = resolve_symlink(&possible_symlink) { - if known_symlinks.contains(&symlink) { - symlinks.push(possible_symlink); - } + let known_symlinks = symlinks.clone(); + let other_symlinks: Vec = symlinks + .par_iter() + .flat_map(|symlink| { + if let Some(bin) = symlink.parent() { + find_executables(bin) + .into_iter() + .filter(|possible_symlink| { + if let Some(resolved) = resolve_symlink(possible_symlink) { + known_symlinks.contains(&resolved) + } else { + false } - } - symlinks - } else { - vec![] - } - }) + }) + .collect::>() + } else { + vec![] + } }) - .collect::>(); - let other_symlinks = threads - .into_iter() - .flat_map(|t| t.join().unwrap()) - .collect::>(); + .collect(); symlinks.extend(other_symlinks); symlinks.sort();