From 2ea1d49316b661953a4638cc9fce601c2caf6643 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 17:25:38 -0600 Subject: [PATCH 1/7] Enhance VFS watch functionality and introduce recording mode - Updated the `watch` method in `VfsBackend` to accept a `recursive` parameter, allowing for more flexible directory watching. - Implemented a recording mode in the `Vfs` struct to track accessed paths without immediately watching them, improving performance during read operations. - Added tests to verify the behavior of the new recording mode and ensure correct path tracking. - Enhanced the `StdBackend` and `NoopBackend` implementations to support the updated watch functionality. These changes aim to improve the VFS's ability to manage file system events and enhance user control over watching behavior. --- crates/memofs/src/in_memory_fs.rs | 2 +- crates/memofs/src/lib.rs | 101 ++++++++++- crates/memofs/src/noop_backend.rs | 2 +- crates/memofs/src/std_backend.rs | 101 ++++++----- src/change_processor.rs | 28 +++- src/serve_session.rs | 37 ++++- src/snapshot/metadata.rs | 3 + src/snapshot_middleware/mod.rs | 268 ++++++++++++++++++++++++++++++ 8 files changed, 486 insertions(+), 56 deletions(-) diff --git a/crates/memofs/src/in_memory_fs.rs b/crates/memofs/src/in_memory_fs.rs index 9ac1041d..2ce1c658 100644 --- a/crates/memofs/src/in_memory_fs.rs +++ b/crates/memofs/src/in_memory_fs.rs @@ -241,7 +241,7 @@ impl VfsBackend for InMemoryFs { inner.event_receiver.clone() } - fn watch(&mut self, _path: &Path) -> io::Result<()> { + fn watch(&mut self, _path: &Path, _recursive: bool) -> io::Result<()> { Ok(()) } diff --git a/crates/memofs/src/lib.rs b/crates/memofs/src/lib.rs index 3109d235..8cb973de 100644 --- a/crates/memofs/src/lib.rs +++ b/crates/memofs/src/lib.rs @@ -22,7 +22,7 @@ mod noop_backend; mod snapshot; mod std_backend; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, MutexGuard}; use std::{io, str}; @@ -94,7 +94,7 @@ pub trait VfsBackend: sealed::Sealed + Send + 'static { fn remove_dir_all(&mut self, path: &Path) -> io::Result<()>; fn event_receiver(&self) -> crossbeam_channel::Receiver; - fn watch(&mut self, path: &Path) -> io::Result<()>; + fn watch(&mut self, path: &Path, recursive: bool) -> io::Result<()>; fn unwatch(&mut self, path: &Path) -> io::Result<()>; } @@ -159,17 +159,28 @@ pub enum VfsEvent { struct VfsInner { backend: Box, watch_enabled: bool, + watch_recursive: bool, prefetch_cache: Option, + recorded_watch_paths: Option>, } impl VfsInner { + fn watch_or_record(&mut self, path: &Path) -> io::Result<()> { + if let Some(ref mut recorded) = self.recorded_watch_paths { + recorded.insert(path.to_path_buf()); + Ok(()) + } else { + self.backend.watch(path, self.watch_recursive) + } + } + /// Read raw bytes from the prefetch cache or the backend. /// Removes the entry from the cache on hit to free memory. fn read_raw(&mut self, path: &Path) -> io::Result> { if let Some(cache) = &mut self.prefetch_cache { if let Some(contents) = cache.files.remove(path) { if self.watch_enabled { - self.backend.watch(path)?; + self.watch_or_record(path)?; } return Ok(contents); } @@ -178,7 +189,7 @@ impl VfsInner { let contents = self.backend.read(path)?; if self.watch_enabled { - self.backend.watch(path)?; + self.watch_or_record(path)?; } Ok(contents) @@ -220,7 +231,7 @@ impl VfsInner { if let Some(cache) = &mut self.prefetch_cache { if let Some(child_paths) = cache.children.remove(path) { if self.watch_enabled { - self.backend.watch(path)?; + self.watch_or_record(path)?; } let inner = child_paths.into_iter().map(|p| Ok(DirEntry { path: p })); return Ok(ReadDir { @@ -232,7 +243,7 @@ impl VfsInner { let dir = self.backend.read_dir(path)?; if self.watch_enabled { - self.backend.watch(path)?; + self.watch_or_record(path)?; } Ok(dir) @@ -345,7 +356,9 @@ impl Vfs { let lock = VfsInner { backend: Box::new(backend), watch_enabled: true, + watch_recursive: true, prefetch_cache: None, + recorded_watch_paths: None, }; Self { @@ -388,6 +401,36 @@ impl Vfs { inner.watch_enabled = enabled; } + /// Sets whether new watches use recursive or non-recursive mode. + /// + /// When false, each `backend.watch()` call only watches the specific + /// directory -- not its children. This allows selective watching where + /// only directories actually traversed during snapshot get OS handles. + pub fn set_watch_recursive(&self, recursive: bool) { + let mut inner = self.inner.lock().unwrap(); + inner.watch_recursive = recursive; + } + + /// Begin recording mode: `read`/`read_dir` calls record touched paths + /// into an internal set instead of calling `backend.watch()`. + pub fn start_watch_recording(&self) { + let mut inner = self.inner.lock().unwrap(); + inner.recorded_watch_paths = Some(HashSet::new()); + } + + /// End recording mode and return the set of paths that were touched. + pub fn take_recorded_paths(&self) -> Option> { + let mut inner = self.inner.lock().unwrap(); + inner.recorded_watch_paths.take() + } + + /// Explicitly watch a path using the current `watch_recursive` setting. + pub fn watch>(&self, path: P) -> io::Result<()> { + let mut inner = self.inner.lock().unwrap(); + let recursive = inner.watch_recursive; + inner.backend.watch(path.as_ref(), recursive) + } + /// Read a file from the VFS, or the underlying backend if it isn't /// resident. /// @@ -767,6 +810,50 @@ mod test { assert_eq!(vfs.read("test").unwrap().as_slice(), b"data"); } + #[test] + fn recording_mode_collects_read_dir_paths() { + let mut imfs = InMemoryFs::new(); + imfs.load_snapshot( + "/root", + VfsSnapshot::dir(HashMap::from([ + ("child.txt", VfsSnapshot::file("hello")), + ( + "subdir", + VfsSnapshot::dir(HashMap::from([("nested.txt", VfsSnapshot::file("world"))])), + ), + ])), + ) + .unwrap(); + + let vfs = Vfs::new(imfs); + vfs.start_watch_recording(); + + let _ = vfs.read_dir("/root").unwrap(); + let _ = vfs.read_dir("/root/subdir").unwrap(); + let _ = vfs.read("/root/child.txt").unwrap(); + + let recorded = vfs.take_recorded_paths().unwrap(); + assert!(recorded.contains(&PathBuf::from("/root"))); + assert!(recorded.contains(&PathBuf::from("/root/subdir"))); + assert!(recorded.contains(&PathBuf::from("/root/child.txt"))); + } + + #[test] + fn recording_mode_returns_none_when_not_started() { + let imfs = InMemoryFs::new(); + let vfs = Vfs::new(imfs); + assert!(vfs.take_recorded_paths().is_none()); + } + + #[test] + fn recording_mode_cleared_after_take() { + let imfs = InMemoryFs::new(); + let vfs = Vfs::new(imfs); + vfs.start_watch_recording(); + let _ = vfs.take_recorded_paths(); + assert!(vfs.take_recorded_paths().is_none()); + } + #[test] fn prefetch_cache_watch_registered_on_cache_hit() { let contents = "hello world".to_string(); @@ -779,7 +866,6 @@ mod test { cache_files.insert(file_path.clone(), contents.as_bytes().to_vec()); vfs.set_prefetch_cache(PrefetchCache { files: cache_files, - canonical: HashMap::new(), is_file: HashMap::new(), children: HashMap::new(), }); @@ -825,7 +911,6 @@ mod test { let vfs = Arc::new(Vfs::new(StdBackend::new_for_testing())); vfs.set_prefetch_cache(PrefetchCache { files: cache_files, - canonical: HashMap::new(), is_file: HashMap::new(), children: HashMap::new(), }); diff --git a/crates/memofs/src/noop_backend.rs b/crates/memofs/src/noop_backend.rs index 98b86d1a..90ac01c3 100644 --- a/crates/memofs/src/noop_backend.rs +++ b/crates/memofs/src/noop_backend.rs @@ -54,7 +54,7 @@ impl VfsBackend for NoopBackend { crossbeam_channel::never() } - fn watch(&mut self, _path: &Path) -> io::Result<()> { + fn watch(&mut self, _path: &Path, _recursive: bool) -> io::Result<()> { Err(io::Error::other("NoopBackend doesn't do anything")) } diff --git a/crates/memofs/src/std_backend.rs b/crates/memofs/src/std_backend.rs index 49f55539..a8bf9fbb 100644 --- a/crates/memofs/src/std_backend.rs +++ b/crates/memofs/src/std_backend.rs @@ -58,6 +58,7 @@ pub struct StdBackend { debouncer: Debouncer, watcher_receiver: Receiver, watches: HashSet, + recursive_watches: HashSet, /// Receiver for critical errors from the watcher thread. /// Callers should poll this to detect when watching becomes unreliable. critical_error_receiver: Receiver, @@ -113,6 +114,7 @@ impl StdBackend { debouncer, watcher_receiver: event_rx, watches: HashSet::new(), + recursive_watches: HashSet::new(), critical_error_receiver: error_rx, } } @@ -328,26 +330,43 @@ impl VfsBackend for StdBackend { self.watcher_receiver.clone() } - fn watch(&mut self, path: &Path) -> io::Result<()> { - if self.watches.contains(path) - || path - .ancestors() - .any(|ancestor| self.watches.contains(ancestor)) + fn watch(&mut self, path: &Path, recursive: bool) -> io::Result<()> { + if self.watches.contains(path) { + return Ok(()); + } + if path + .ancestors() + .skip(1) + .any(|ancestor| self.recursive_watches.contains(ancestor)) { - Ok(()) + return Ok(()); + } + + let mode = if recursive { + RecursiveMode::Recursive } else { - // Only add to watches AFTER the watch succeeds - // This prevents a failed watch from permanently marking the path as "watched" - match self.debouncer.watch(path, RecursiveMode::Recursive) { - Ok(()) => { - log::debug!("Watching path: {}", path.display()); - self.watches.insert(path.to_path_buf()); - Ok(()) - } - Err(err) => { - log::warn!("Failed to watch path {}: {:?}", path.display(), err); - Err(io::Error::other(format!("{:?}", err))) + RecursiveMode::NonRecursive + }; + match self.debouncer.watch(path, mode) { + Ok(()) => { + log::debug!( + "Watching path ({}): {}", + if recursive { + "recursive" + } else { + "non-recursive" + }, + path.display() + ); + self.watches.insert(path.to_path_buf()); + if recursive { + self.recursive_watches.insert(path.to_path_buf()); } + Ok(()) + } + Err(err) => { + log::warn!("Failed to watch path {}: {:?}", path.display(), err); + Err(io::Error::other(format!("{:?}", err))) } } } @@ -366,6 +385,7 @@ impl VfsBackend for StdBackend { ); } self.watches.remove(path); + self.recursive_watches.remove(path); Ok(()) } Err(err) => { @@ -382,6 +402,7 @@ impl VfsBackend for StdBackend { path.display() ); self.watches.remove(path); + self.recursive_watches.remove(path); Ok(()) } else { log::warn!("Failed to unwatch path {}: {:?}", path.display(), err); @@ -414,17 +435,17 @@ mod tests { let mut backend = StdBackend::new_for_testing(); // Watch should succeed and add to internal set - assert!(backend.watch(&file_path).is_ok()); + assert!(backend.watch(&file_path, true).is_ok()); // Watching again should be a no-op (already watched) - assert!(backend.watch(&file_path).is_ok()); + assert!(backend.watch(&file_path, true).is_ok()); // Watch a non-existent path - behavior varies by platform // On some systems, watching a non-existent file might succeed // because notify watches the parent directory. We just verify // that calling watch doesn't panic or corrupt state. let nonexistent = dir.path().join("nonexistent.txt"); - let _ = backend.watch(&nonexistent); // Result varies by platform + let _ = backend.watch(&nonexistent, true); // Result varies by platform } #[test] @@ -451,13 +472,13 @@ mod tests { let mut backend = StdBackend::new_for_testing(); // Watch the file - assert!(backend.watch(&file_path).is_ok()); + assert!(backend.watch(&file_path, true).is_ok()); // Unwatch should succeed assert!(backend.unwatch(&file_path).is_ok()); // Should be able to watch again after unwatch - assert!(backend.watch(&file_path).is_ok()); + assert!(backend.watch(&file_path, true).is_ok()); } #[test] @@ -471,10 +492,10 @@ mod tests { let mut backend = StdBackend::new_for_testing(); // Watch the parent directory - assert!(backend.watch(&subdir).is_ok()); + assert!(backend.watch(&subdir, true).is_ok()); // Watching a file inside should be a no-op (covered by parent) - assert!(backend.watch(&file_path).is_ok()); + assert!(backend.watch(&file_path, true).is_ok()); } #[test] @@ -504,12 +525,12 @@ mod tests { // Rapidly cycle watch/unwatch to stress test state consistency for _ in 0..10 { - assert!(backend.watch(&file_path).is_ok()); + assert!(backend.watch(&file_path, true).is_ok()); assert!(backend.unwatch(&file_path).is_ok()); } // Final watch should still work - assert!(backend.watch(&file_path).is_ok()); + assert!(backend.watch(&file_path, true).is_ok()); } #[test] @@ -522,7 +543,7 @@ mod tests { let event_rx = backend.event_receiver(); // Watch the directory (more reliable than watching the file directly) - assert!(backend.watch(dir.path()).is_ok()); + assert!(backend.watch(dir.path(), true).is_ok()); // Give the watcher time to start std::thread::sleep(Duration::from_millis(100)); @@ -597,7 +618,7 @@ mod tests { let mut backend = StdBackend::new_for_testing(); let event_rx = backend.event_receiver(); - assert!(backend.watch(dir.path()).is_ok()); + assert!(backend.watch(dir.path(), true).is_ok()); std::thread::sleep(Duration::from_millis(100)); // Rapid modifications - 20 writes in quick succession @@ -635,7 +656,7 @@ mod tests { let mut backend = StdBackend::new_for_testing(); let event_rx = backend.event_receiver(); - assert!(backend.watch(dir.path()).is_ok()); + assert!(backend.watch(dir.path(), true).is_ok()); std::thread::sleep(Duration::from_millis(100)); // 10 cycles of create/delete @@ -670,7 +691,7 @@ mod tests { let dir = tempdir().unwrap(); let mut backend = StdBackend::new_for_testing(); let event_rx = backend.event_receiver(); - assert!(backend.watch(dir.path()).is_ok()); + assert!(backend.watch(dir.path(), true).is_ok()); std::thread::sleep(Duration::from_millis(100)); // Create 20 files rapidly @@ -722,7 +743,7 @@ mod tests { let mut backend = StdBackend::new_for_testing(); let event_rx = backend.event_receiver(); - assert!(backend.watch(dir.path()).is_ok()); + assert!(backend.watch(dir.path(), true).is_ok()); std::thread::sleep(Duration::from_millis(100)); // Rename the file @@ -758,7 +779,7 @@ mod tests { let mut backend = StdBackend::new_for_testing(); let event_rx = backend.event_receiver(); - assert!(backend.watch(dir.path()).is_ok()); + assert!(backend.watch(dir.path(), true).is_ok()); std::thread::sleep(Duration::from_millis(100)); // Chain of renames @@ -781,7 +802,7 @@ mod tests { let dir = tempdir().unwrap(); let mut backend = StdBackend::new_for_testing(); let event_rx = backend.event_receiver(); - assert!(backend.watch(dir.path()).is_ok()); + assert!(backend.watch(dir.path(), true).is_ok()); std::thread::sleep(Duration::from_millis(100)); // Create nested structure @@ -826,7 +847,7 @@ mod tests { let mut backend = StdBackend::new_for_testing(); let event_rx = backend.event_receiver(); - assert!(backend.watch(dir.path()).is_ok()); + assert!(backend.watch(dir.path(), true).is_ok()); std::thread::sleep(Duration::from_millis(100)); // 100 writes with no delay @@ -863,7 +884,7 @@ mod tests { let dir = tempdir().unwrap(); let mut backend = StdBackend::new_for_testing(); let event_rx = backend.event_receiver(); - assert!(backend.watch(dir.path()).is_ok()); + assert!(backend.watch(dir.path(), true).is_ok()); std::thread::sleep(Duration::from_millis(100)); let files: Vec<_> = (0..5) @@ -915,7 +936,7 @@ mod tests { let mut backend = StdBackend::new_for_testing(); let event_rx = backend.event_receiver(); - assert!(backend.watch(dir.path()).is_ok()); + assert!(backend.watch(dir.path(), true).is_ok()); std::thread::sleep(Duration::from_millis(100)); // Simulate multiple undo/redo cycles @@ -969,7 +990,7 @@ mod tests { let dir = tempdir().unwrap(); let mut backend = StdBackend::new_for_testing(); let event_rx = backend.event_receiver(); - assert!(backend.watch(dir.path()).is_ok()); + assert!(backend.watch(dir.path(), true).is_ok()); std::thread::sleep(Duration::from_millis(100)); // Rapidly create subdirectories with files @@ -1005,7 +1026,7 @@ mod tests { let mut backend = StdBackend::new_for_testing(); let event_rx = backend.event_receiver(); - assert!(backend.watch(dir.path()).is_ok()); + assert!(backend.watch(dir.path(), true).is_ok()); std::thread::sleep(Duration::from_millis(100)); // Create empty file @@ -1041,7 +1062,7 @@ mod tests { let mut backend = StdBackend::new_for_testing(); let event_rx = backend.event_receiver(); - assert!(backend.watch(dir.path()).is_ok()); + assert!(backend.watch(dir.path(), true).is_ok()); std::thread::sleep(Duration::from_millis(100)); fs_err::write(&file_path, "-- long name test").unwrap(); @@ -1067,7 +1088,7 @@ mod tests { let mut backend = StdBackend::new_for_testing(); let event_rx = backend.event_receiver(); - assert!(backend.watch(dir.path()).is_ok()); + assert!(backend.watch(dir.path(), true).is_ok()); std::thread::sleep(Duration::from_millis(100)); for name in &special_files { diff --git a/src/change_processor.rs b/src/change_processor.rs index fbe41b20..4d3b05db 100644 --- a/src/change_processor.rs +++ b/src/change_processor.rs @@ -16,7 +16,7 @@ use crate::{ snapshot::{ apply_patch_set, compute_patch_set, AppliedPatchSet, InstigatingSource, PatchSet, RojoTree, }, - snapshot_middleware::{snapshot_from_vfs, snapshot_project_node}, + snapshot_middleware::{is_script_relevant_path, snapshot_from_vfs, snapshot_project_node}, syncback::{ dedup_suffix::{compute_cleanup_action, parse_dedup_suffix, DedupCleanupAction}, deduplicate_name, name_needs_slugify, slugify_name, strip_script_suffix, @@ -92,6 +92,7 @@ impl ChangeProcessor { project_file_path: PathBuf, critical_error_receiver: Option>, git_repo_root: Option, + sync_scripts_only: bool, ) -> Self { let (shutdown_sender, shutdown_receiver) = crossbeam_channel::bounded(1); let vfs_receiver = vfs.event_receiver(); @@ -109,6 +110,7 @@ impl ChangeProcessor { project_file_path, ref_path_index, git_repo_root, + sync_scripts_only, }; let job_thread = jod_thread::Builder::new() @@ -268,6 +270,9 @@ struct JobThreadContext { /// Git repository root, if the project is in a git repo. /// Used for auto-staging Source writes. git_repo_root: Option, + + /// When true, only script-related VFS events are processed. + sync_scripts_only: bool, } impl JobThreadContext { @@ -723,6 +728,24 @@ impl JobThreadContext { } } + if self.sync_scripts_only { + if let Some(ref path) = event_path { + if path.is_file() { + let not_in_tree = { + let tree = self.tree.lock().unwrap(); + tree.get_ids_at_path(path).is_empty() + }; + if not_in_tree && !is_script_relevant_path(path) { + log::trace!( + "VFS event SKIPPED (scripts-only, non-script file): {}", + self.display_path(path) + ); + return Vec::new(); + } + } + } + } + // For a given VFS event, we might have many changes to different parts // of the tree. Calculate and apply all of these changes. let applied_patches = match event { @@ -890,7 +913,8 @@ impl JobThreadContext { use crate::snapshot::InstanceContext; let start = Instant::now(); - let instance_context = InstanceContext::new(); + let mut instance_context = InstanceContext::new(); + instance_context.sync_scripts_only = self.sync_scripts_only; let snapshot = match snapshot_from_vfs(&instance_context, &self.vfs, &self.project_file_path) { diff --git a/src/serve_session.rs b/src/serve_session.rs index 1ef4628d..c04afa92 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -19,7 +19,7 @@ use crate::{ apply_patch_set, compute_patch_set, AppliedPatchSet, InstanceContext, InstanceSnapshot, PatchSet, RojoTree, }, - snapshot_middleware::snapshot_from_vfs, + snapshot_middleware::{is_script_relevant_path, snapshot_from_vfs}, }; /// Set to `true` to validate on plugin connect (useful for testing, do not enable on production). @@ -131,7 +131,7 @@ pub struct ServeSession { /// Collect all filesystem paths reachable from the project tree's `$path` /// entries, then read file contents in parallel. -fn prefetch_project_files(project: &Project) -> io::Result { +fn prefetch_project_files(project: &Project, sync_scripts_only: bool) -> io::Result { use rayon::prelude::*; use std::collections::HashMap; use walkdir::WalkDir; @@ -200,6 +200,7 @@ fn prefetch_project_files(project: &Project) -> io::Result { let file_data: Vec<_> = entries .par_iter() .filter(|e| e.file_type().is_file()) + .filter(|e| !sync_scripts_only || is_script_relevant_path(e.path())) .filter_map(|e| { let path = e.path().to_path_buf(); std::fs::read(&path).ok().map(|c| (path, c)) @@ -266,10 +267,11 @@ impl ServeSession { log::trace!("Starting new ServeSession at path {}", start_path.display()); let root_project = Project::load_initial_project(vfs, start_path)?; + let sync_scripts_only = root_project.sync_scripts_only.unwrap_or(false); if std::env::var("ATLAS_SEQUENTIAL").is_err() { let prefetch_start = Instant::now(); - match prefetch_project_files(&root_project) { + match prefetch_project_files(&root_project, sync_scripts_only) { Ok(cache) => { log::debug!( "Prefetch total: {} files in {:.1?}", @@ -286,7 +288,11 @@ impl ServeSession { let mut tree = RojoTree::new(InstanceSnapshot::new()); let root_id = tree.get_root_id(); - let instance_context = InstanceContext::new(); + let mut instance_context = InstanceContext::new(); + instance_context.sync_scripts_only = sync_scripts_only; + + vfs.set_watch_recursive(false); + vfs.start_watch_recording(); let snap_start = Instant::now(); log::trace!("Generating snapshot of instances from VFS"); @@ -303,6 +309,28 @@ impl ServeSession { apply_patch_set(&mut tree, patch_set); log::debug!("Patch computed + applied in {:.1?}", patch_start.elapsed()); + if let Some(recorded_paths) = vfs.take_recorded_paths() { + let dir_count = recorded_paths.iter().filter(|p| p.is_dir()).count(); + for path in &recorded_paths { + if path.is_dir() { + let _ = vfs.watch(path); + } + } + let mut path_roots = Vec::new(); + collect_path_roots( + &root_project.tree, + root_project.folder_location(), + &mut path_roots, + ); + for root in &path_roots { + let _ = vfs.watch(root); + } + log::debug!( + "Selective watching: {} directories watched (non-recursive)", + dir_count + path_roots.len(), + ); + } + Ok((root_project, tree)) } @@ -356,6 +384,7 @@ impl ServeSession { root_project.file_location.clone(), critical_error_receiver, git_repo_root.clone(), + root_project.sync_scripts_only.unwrap_or(false), ); Ok(Self { diff --git a/src/snapshot/metadata.rs b/src/snapshot/metadata.rs index 82612f30..9fb96cce 100644 --- a/src/snapshot/metadata.rs +++ b/src/snapshot/metadata.rs @@ -155,6 +155,8 @@ pub struct InstanceContext { pub path_ignore_rules: Arc>, #[serde(skip_serializing_if = "Vec::is_empty")] pub sync_rules: Vec, + #[serde(skip)] + pub sync_scripts_only: bool, } impl InstanceContext { @@ -162,6 +164,7 @@ impl InstanceContext { Self { path_ignore_rules: Arc::new(Vec::new()), sync_rules: Vec::new(), + sync_scripts_only: false, } } diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index f8ca439e..077e7f12 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -79,6 +79,13 @@ pub fn snapshot_from_vfs( // If and when we do, make sure to go support it in // `Project::set_file_name`, as right now it special-cases // `default.project.json5` as an `init` path. + if context.sync_scripts_only + && !middleware.is_script() + && middleware != Middleware::Dir + && middleware != Middleware::Project + { + return Ok(None); + } match middleware { Middleware::Dir => middleware.snapshot(context, vfs, path, dir_name), _ => middleware.snapshot(context, vfs, &init_path, dir_name), @@ -168,11 +175,23 @@ fn snapshot_from_path( // name is needed (e.g. for names with special chars), it comes from the // `name` field in adjacent .meta.json / .model.json files. if let Some(rule) = context.get_user_sync_rule(path) { + if context.sync_scripts_only + && !rule.middleware.is_script() + && rule.middleware != Middleware::Project + { + return Ok(None); + } let name = rule.file_name_for_path(path)?; return rule.middleware.snapshot(context, vfs, path, name); } else { for rule in default_sync_rules() { if rule.matches(path) { + if context.sync_scripts_only + && !rule.middleware.is_script() + && rule.middleware != Middleware::Project + { + return Ok(None); + } let name = rule.file_name_for_path(path)?; return rule.middleware.snapshot(context, vfs, path, name); } @@ -319,6 +338,27 @@ impl Middleware { } } + /// Returns whether this middleware produces a script instance + /// (Script, LocalScript, or ModuleScript). + #[inline] + pub fn is_script(&self) -> bool { + matches!( + self, + Self::ServerScript + | Self::ClientScript + | Self::ModuleScript + | Self::PluginScript + | Self::LocalScript + | Self::LegacyScript + | Self::ServerScriptDir + | Self::ClientScriptDir + | Self::ModuleScriptDir + | Self::PluginScriptDir + | Self::LocalScriptDir + | Self::LegacyScriptDir + ) + } + /// Returns whether this particular middleware would become a directory. #[inline] pub fn is_dir(&self) -> bool { @@ -454,3 +494,231 @@ pub fn default_sync_rules() -> &'static [SyncRule] { ] }) } + +/// Returns whether a filesystem path is relevant in scripts-only mode. +/// +/// Matches script files (`.luau`, `.lua`), meta files (`.meta.json5`, +/// `.meta.json`), and project files (`.project.json5`, `.project.json`). +pub fn is_script_relevant_path(path: &Path) -> bool { + let name = match path.file_name().and_then(|n| n.to_str()) { + Some(n) => n.to_lowercase(), + None => return false, + }; + name.ends_with(".luau") + || name.ends_with(".lua") + || name.ends_with(".meta.json5") + || name.ends_with(".meta.json") + || name.ends_with(".project.json5") + || name.ends_with(".project.json") +} + +#[cfg(test)] +mod test { + use super::*; + use std::collections::HashMap; + + use memofs::{InMemoryFs, VfsSnapshot}; + + #[test] + fn is_script_covers_all_script_types() { + assert!(Middleware::ServerScript.is_script()); + assert!(Middleware::ClientScript.is_script()); + assert!(Middleware::ModuleScript.is_script()); + assert!(Middleware::PluginScript.is_script()); + assert!(Middleware::LocalScript.is_script()); + assert!(Middleware::LegacyScript.is_script()); + assert!(Middleware::ServerScriptDir.is_script()); + assert!(Middleware::ClientScriptDir.is_script()); + assert!(Middleware::ModuleScriptDir.is_script()); + assert!(Middleware::PluginScriptDir.is_script()); + assert!(Middleware::LocalScriptDir.is_script()); + assert!(Middleware::LegacyScriptDir.is_script()); + } + + #[test] + fn is_script_excludes_non_scripts() { + assert!(!Middleware::Dir.is_script()); + assert!(!Middleware::Project.is_script()); + assert!(!Middleware::Csv.is_script()); + assert!(!Middleware::JsonModel.is_script()); + assert!(!Middleware::Json.is_script()); + assert!(!Middleware::Rbxm.is_script()); + assert!(!Middleware::Rbxmx.is_script()); + assert!(!Middleware::Toml.is_script()); + assert!(!Middleware::Text.is_script()); + assert!(!Middleware::Yaml.is_script()); + assert!(!Middleware::Ignore.is_script()); + assert!(!Middleware::CsvDir.is_script()); + } + + #[test] + fn scripts_only_skips_non_script_file() { + let mut imfs = InMemoryFs::new(); + imfs.load_snapshot( + "/project", + VfsSnapshot::dir(HashMap::from([ + ("data.json5", VfsSnapshot::file("{}")), + ("script.server.luau", VfsSnapshot::file("print('hi')")), + ])), + ) + .unwrap(); + + let vfs = Vfs::new(imfs); + let mut context = InstanceContext::new(); + context.sync_scripts_only = true; + + let json_result = + snapshot_from_vfs(&context, &vfs, Path::new("/project/data.json5")).unwrap(); + assert!(json_result.is_none()); + + let script_result = + snapshot_from_vfs(&context, &vfs, Path::new("/project/script.server.luau")).unwrap(); + assert!(script_result.is_some()); + assert_eq!(script_result.unwrap().class_name.as_str(), "Script"); + } + + #[test] + fn scripts_only_preserves_directories() { + let mut imfs = InMemoryFs::new(); + imfs.load_snapshot( + "/project", + VfsSnapshot::dir(HashMap::from([( + "models", + VfsSnapshot::dir(HashMap::from([( + "part.rbxm", + VfsSnapshot::file(b"\x00".as_ref()), + )])), + )])), + ) + .unwrap(); + + let vfs = Vfs::new(imfs); + let mut context = InstanceContext::new(); + context.sync_scripts_only = true; + + let result = snapshot_from_vfs(&context, &vfs, Path::new("/project/models")).unwrap(); + assert!(result.is_some()); + let snapshot = result.unwrap(); + assert_eq!(snapshot.class_name.as_str(), "Folder"); + assert!(snapshot.children.is_empty()); + } + + #[test] + fn scripts_only_allows_project_files() { + let mut imfs = InMemoryFs::new(); + imfs.load_snapshot( + "/nested.project.json5", + VfsSnapshot::file(r#"{"name": "Nested", "tree": {"$className": "Folder"}}"#), + ) + .unwrap(); + + let vfs = Vfs::new(imfs); + let mut context = InstanceContext::new(); + context.sync_scripts_only = true; + + let result = snapshot_from_vfs(&context, &vfs, Path::new("/nested.project.json5")).unwrap(); + assert!(result.is_some()); + } + + #[test] + fn normal_mode_includes_all_files() { + let mut imfs = InMemoryFs::new(); + imfs.load_snapshot( + "/project", + VfsSnapshot::dir(HashMap::from([ + ("data.json5", VfsSnapshot::file("{}")), + ("script.server.luau", VfsSnapshot::file("print('hi')")), + ])), + ) + .unwrap(); + + let vfs = Vfs::new(imfs); + let context = InstanceContext::new(); + + let json_result = + snapshot_from_vfs(&context, &vfs, Path::new("/project/data.json5")).unwrap(); + assert!(json_result.is_some()); + + let script_result = + snapshot_from_vfs(&context, &vfs, Path::new("/project/script.server.luau")).unwrap(); + assert!(script_result.is_some()); + } + + #[test] + fn is_script_relevant_path_accepts_scripts_and_meta() { + assert!(is_script_relevant_path(Path::new("/src/main.luau"))); + assert!(is_script_relevant_path(Path::new("/src/init.server.luau"))); + assert!(is_script_relevant_path(Path::new("/src/old.lua"))); + assert!(is_script_relevant_path(Path::new("/src/old.server.lua"))); + assert!(is_script_relevant_path(Path::new("/src/file.meta.json5"))); + assert!(is_script_relevant_path(Path::new("/src/file.meta.json"))); + assert!(is_script_relevant_path(Path::new("/nested.project.json5"))); + assert!(is_script_relevant_path(Path::new("/nested.project.json"))); + } + + #[test] + fn is_script_relevant_path_rejects_non_scripts() { + assert!(!is_script_relevant_path(Path::new("/src/data.json5"))); + assert!(!is_script_relevant_path(Path::new("/src/model.rbxm"))); + assert!(!is_script_relevant_path(Path::new("/src/notes.txt"))); + assert!(!is_script_relevant_path(Path::new("/src/table.csv"))); + assert!(!is_script_relevant_path(Path::new("/src/config.toml"))); + assert!(!is_script_relevant_path(Path::new("/src/data.yaml"))); + assert!(!is_script_relevant_path(Path::new("/src/model.rbxmx"))); + assert!(!is_script_relevant_path(Path::new("/src"))); + assert!(!is_script_relevant_path(Path::new("/src/no_ext"))); + } + + #[test] + fn scripts_only_filters_csv_dir() { + let mut imfs = InMemoryFs::new(); + imfs.load_snapshot( + "/project", + VfsSnapshot::dir(HashMap::from([( + "localization", + VfsSnapshot::dir(HashMap::from([( + "init.csv", + VfsSnapshot::file("Key,Source,Example\nkey1,Hello,Hello"), + )])), + )])), + ) + .unwrap(); + + let vfs = Vfs::new(imfs); + let mut context = InstanceContext::new(); + context.sync_scripts_only = true; + + let result = snapshot_from_vfs(&context, &vfs, Path::new("/project/localization")).unwrap(); + assert!( + result.is_none(), + "CsvDir should be filtered in scripts-only mode" + ); + } + + #[test] + fn scripts_only_preserves_script_dir() { + let mut imfs = InMemoryFs::new(); + imfs.load_snapshot( + "/project", + VfsSnapshot::dir(HashMap::from([( + "MyScript", + VfsSnapshot::dir(HashMap::from([( + "init.server.luau", + VfsSnapshot::file("print('hello')"), + )])), + )])), + ) + .unwrap(); + + let vfs = Vfs::new(imfs); + let mut context = InstanceContext::new(); + context.sync_scripts_only = true; + + let result = snapshot_from_vfs(&context, &vfs, Path::new("/project/MyScript")).unwrap(); + assert!( + result.is_some(), + "Script dir should pass through in scripts-only mode" + ); + assert_eq!(result.unwrap().class_name.as_str(), "Script"); + } +} From 02638535ec2a580f3c175f6ecffcd442cb3f52f1 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 17:29:29 -0600 Subject: [PATCH 2/7] Add VFS recording mode and enhance git configuration - Introduced VFS recording mode and non-recursive watch support to optimize file-watching during `atlas serve` startup for large projects. - Enhanced scripts-only mode to filter out non-script files at various processing layers. - Configured `core.longpaths` in git during repository initialization to support long file paths on Windows. These updates improve performance and usability for users managing large projects. --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e44aed9..8be9b575 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,18 @@ Making a new release? Simply add the new header with the version and date undern ## Unreleased +* Add VFS recording mode and non-recursive watch support to reduce OS file-watch handles during `atlas serve` startup on large projects +* Enhance scripts-only mode to filter non-script files at the snapshot middleware, VFS event, and prefetch layers +* Set `core.longpaths` git config during repository initialization for Windows long-path support + +
+Full commit log + +- `2ea1d493` Enhance VFS watch functionality and introduce recording mode +- `bfb5815f` Update TODO and enhance git configuration + +
+ ## [8.5.2] (March 3rd, 2026) * Fix `atlas init` to commit project files before adding agent submodules, producing cleaner git history From 9ddd9379abf4c874481d0790373af57a74de9379 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 17:43:08 -0600 Subject: [PATCH 3/7] Refine VFS watch behavior to support recursive watching - Updated the `read_raw` method in `VfsInner` to ensure that the file watching behavior is only triggered when both `watch_enabled` and `watch_recursive` are true. This change enhances the control over file watching, particularly in scenarios involving nested directories. These modifications aim to improve the efficiency and accuracy of the VFS's file watching capabilities. --- crates/memofs/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/memofs/src/lib.rs b/crates/memofs/src/lib.rs index 8cb973de..12ecd08e 100644 --- a/crates/memofs/src/lib.rs +++ b/crates/memofs/src/lib.rs @@ -179,7 +179,7 @@ impl VfsInner { fn read_raw(&mut self, path: &Path) -> io::Result> { if let Some(cache) = &mut self.prefetch_cache { if let Some(contents) = cache.files.remove(path) { - if self.watch_enabled { + if self.watch_enabled && self.watch_recursive { self.watch_or_record(path)?; } return Ok(contents); @@ -188,7 +188,7 @@ impl VfsInner { let contents = self.backend.read(path)?; - if self.watch_enabled { + if self.watch_enabled && self.watch_recursive { self.watch_or_record(path)?; } From a1825f7c4aaa8c3e2e0aa9815087b5d5eb9a7a4c Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 17:49:50 -0600 Subject: [PATCH 4/7] Enhance VFS watch functionality to support upgrade from non-recursive to recursive watching - Modified the `watch` method in `StdBackend` to allow upgrading a non-recursive watch to a recursive one, improving flexibility in file watching. - Added a new test to verify that a non-recursive watch can be upgraded to recursive, ensuring correct behavior in directory watching scenarios. These changes aim to enhance the VFS's ability to manage file system events more effectively. --- crates/memofs/src/std_backend.rs | 39 +++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/crates/memofs/src/std_backend.rs b/crates/memofs/src/std_backend.rs index a8bf9fbb..f3c2c145 100644 --- a/crates/memofs/src/std_backend.rs +++ b/crates/memofs/src/std_backend.rs @@ -332,7 +332,11 @@ impl VfsBackend for StdBackend { fn watch(&mut self, path: &Path, recursive: bool) -> io::Result<()> { if self.watches.contains(path) { - return Ok(()); + if !recursive || self.recursive_watches.contains(path) { + return Ok(()); + } + // Upgrade: path was watched non-recursively, now needs recursive. + // Fall through to re-issue the watch with RecursiveMode::Recursive. } if path .ancestors() @@ -498,6 +502,39 @@ mod tests { assert!(backend.watch(&file_path, true).is_ok()); } + #[test] + fn non_recursive_watch_upgrades_to_recursive() { + let dir = tempdir().unwrap(); + let subdir = dir.path().join("subdir"); + fs_err::create_dir(&subdir).unwrap(); + let child = subdir.join("child.txt"); + fs_err::write(&child, "content").unwrap(); + + let mut backend = StdBackend::new_for_testing(); + + // Watch non-recursively first + assert!(backend.watch(&subdir, false).is_ok()); + assert!(backend.watches.contains(subdir.as_path())); + assert!(!backend.recursive_watches.contains(subdir.as_path())); + + // Child should NOT be covered (parent is non-recursive) + assert!(!subdir + .ancestors() + .skip(1) + .any(|a| backend.recursive_watches.contains(a))); + + // Upgrade to recursive + assert!(backend.watch(&subdir, true).is_ok()); + assert!(backend.recursive_watches.contains(subdir.as_path())); + + // Now a child path should be recognized as covered by the recursive ancestor + assert!(child + .ancestors() + .skip(1) + .any(|a| backend.recursive_watches.contains(a))); + assert!(backend.watch(&child, true).is_ok()); + } + #[test] fn critical_error_receiver_works() { let error_received = Arc::new(AtomicBool::new(false)); From a1b65dbb524822f99f30d7fa77e8d57a21f30cc2 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 17:56:04 -0600 Subject: [PATCH 5/7] Enhance logging for VFS directory watching - Added timing measurement for the selective watching process in the `ServeSession` implementation, improving visibility into performance during directory watching. - Updated the debug log message to include the elapsed time for better monitoring of the watching behavior. These changes aim to provide more detailed insights into the VFS's directory watching performance. --- src/serve_session.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/serve_session.rs b/src/serve_session.rs index c04afa92..587f2456 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -309,6 +309,7 @@ impl ServeSession { apply_patch_set(&mut tree, patch_set); log::debug!("Patch computed + applied in {:.1?}", patch_start.elapsed()); + let watch_start = Instant::now(); if let Some(recorded_paths) = vfs.take_recorded_paths() { let dir_count = recorded_paths.iter().filter(|p| p.is_dir()).count(); for path in &recorded_paths { @@ -326,8 +327,9 @@ impl ServeSession { let _ = vfs.watch(root); } log::debug!( - "Selective watching: {} directories watched (non-recursive)", + "Selective watching: {} directories watched (non-recursive) in {:.1?}", dir_count + path_roots.len(), + watch_start.elapsed(), ); } From 0fb5c8776e5aa4663583f37580e3d50b365c0002 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 18:25:21 -0600 Subject: [PATCH 6/7] Refactor VFS watching mechanism for improved performance - Replaced the selective watching logic with a more efficient recursive watching setup in the `ServeSession` implementation. - Enhanced logging to include the number of recursive watches set up and the elapsed time for the setup process, providing better insights into performance. - Removed the previous recorded paths logic to streamline the watching process. These changes aim to optimize the VFS's directory watching capabilities and improve monitoring during the setup phase. --- src/serve_session.rs | 42 ++++++++++++++++-------------------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/src/serve_session.rs b/src/serve_session.rs index 587f2456..cb7c8c9f 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -291,8 +291,22 @@ impl ServeSession { let mut instance_context = InstanceContext::new(); instance_context.sync_scripts_only = sync_scripts_only; - vfs.set_watch_recursive(false); - vfs.start_watch_recording(); + let watch_start = Instant::now(); + let mut path_roots = Vec::new(); + collect_path_roots( + &root_project.tree, + root_project.folder_location(), + &mut path_roots, + ); + let _ = vfs.watch(root_project.folder_location()); + for root in &path_roots { + let _ = vfs.watch(root); + } + log::debug!( + "Set up {} recursive watches in {:.1?}", + path_roots.len() + 1, + watch_start.elapsed(), + ); let snap_start = Instant::now(); log::trace!("Generating snapshot of instances from VFS"); @@ -309,30 +323,6 @@ impl ServeSession { apply_patch_set(&mut tree, patch_set); log::debug!("Patch computed + applied in {:.1?}", patch_start.elapsed()); - let watch_start = Instant::now(); - if let Some(recorded_paths) = vfs.take_recorded_paths() { - let dir_count = recorded_paths.iter().filter(|p| p.is_dir()).count(); - for path in &recorded_paths { - if path.is_dir() { - let _ = vfs.watch(path); - } - } - let mut path_roots = Vec::new(); - collect_path_roots( - &root_project.tree, - root_project.folder_location(), - &mut path_roots, - ); - for root in &path_roots { - let _ = vfs.watch(root); - } - log::debug!( - "Selective watching: {} directories watched (non-recursive) in {:.1?}", - dir_count + path_roots.len(), - watch_start.elapsed(), - ); - } - Ok((root_project, tree)) } From 14d4bc134d5c3c80668bb83ec42a86fcd97efb9a Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 18:32:13 -0600 Subject: [PATCH 7/7] Update instance context in ServeSession for script synchronization - Modified the `check_tree_freshness` method to set `sync_scripts_only` in the `InstanceContext`, ensuring that the correct synchronization behavior is applied during tree freshness checks. - This change enhances the functionality of the `ServeSession` by allowing for more precise control over script synchronization. --- src/serve_session.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/serve_session.rs b/src/serve_session.rs index cb7c8c9f..60306ad0 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -547,7 +547,8 @@ impl ServeSession { pub fn check_tree_freshness(&self) -> TreeFreshnessReport { let start = Instant::now(); let start_path: &Path = &self.root_project.file_location; - let instance_context = InstanceContext::new(); + let mut instance_context = InstanceContext::new(); + instance_context.sync_scripts_only = self.sync_scripts_only(); let snapshot = match snapshot_from_vfs(&instance_context, &self.vfs, start_path) { Ok(s) => s, @@ -610,7 +611,8 @@ impl ServeSession { let start = Instant::now(); let start_path: &Path = &self.root_project.file_location; - let instance_context = InstanceContext::new(); + let mut instance_context = InstanceContext::new(); + instance_context.sync_scripts_only = self.sync_scripts_only(); let snapshot = match snapshot_from_vfs(&instance_context, &self.vfs, start_path) { Ok(s) => s,