From 6168597acb6b4963e1ed85a192947e86f27a1404 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 19:58:07 -0600 Subject: [PATCH 01/12] Enhance PrefetchCache with directory init-file resolution - Added a new field `dir_init` to `PrefetchCache` to store pre-resolved init-file information for directories, allowing for efficient retrieval during prefetch operations. - Introduced the `prefetch_dir_init` method in the `Vfs` struct to look up init-file info from the prefetch cache. - Updated the `prefetch_project_files` function to populate the `dir_init` map based on the defined `INIT_FILE_PRIORITY`. - Adjusted related tests to ensure proper initialization of the `dir_init` field in various scenarios. These changes improve the performance of directory handling in the VFS by reducing the need for repeated metadata probes. --- crates/memofs/src/lib.rs | 25 ++++++++ src/serve_session.rs | 104 ++++++++++++++++++++++----------- src/snapshot/patch_compute.rs | 13 ++++- src/snapshot_middleware/mod.rs | 63 +++++++++++--------- tests/tests/build.rs | 20 ++++++- 5 files changed, 159 insertions(+), 66 deletions(-) diff --git a/crates/memofs/src/lib.rs b/crates/memofs/src/lib.rs index 12ecd08e..48be421f 100644 --- a/crates/memofs/src/lib.rs +++ b/crates/memofs/src/lib.rs @@ -44,6 +44,12 @@ pub struct PrefetchCache { pub is_file: HashMap, /// Directory path -> sorted child paths. Consumed once per directory. pub children: HashMap>, + /// Pre-resolved init-file info for each directory. The value is `None` + /// when the directory has no init file (plain `Folder`), or + /// `Some((init_file_name, init_path))` for the winning init entry. + /// Populated during prefetch so that `get_dir_middleware` can skip + /// per-file VFS metadata probes entirely. + pub dir_init: HashMap>, } mod sealed { @@ -282,6 +288,10 @@ impl VfsInner { if let Some(&is_file) = cache.is_file.get(path) { return Ok(Metadata { is_file }); } + return Err(io::Error::new( + io::ErrorKind::NotFound, + "not in prefetch cache", + )); } self.backend.metadata(path) @@ -384,6 +394,17 @@ impl Vfs { inner.prefetch_cache = None; } + /// Look up pre-resolved init-file info for a directory from the + /// prefetch cache. Returns `Some(Some((name, path)))` when the + /// directory was found in the cache with an init file, `Some(None)` + /// when the directory was found but has no init file, and `None` + /// when there is no prefetch cache or the directory is not in it. + pub fn prefetch_dir_init(&self, dir: &Path) -> Option> { + let inner = self.inner.lock().unwrap(); + let cache = inner.prefetch_cache.as_ref()?; + cache.dir_init.get(dir).cloned() + } + /// Manually lock the Vfs, useful for large batches of operations. pub fn lock(&self) -> VfsLock<'_> { VfsLock { @@ -732,6 +753,7 @@ mod test { .collect(), is_file: HashMap::new(), children: HashMap::new(), + dir_init: HashMap::new(), } } @@ -868,6 +890,7 @@ mod test { files: cache_files, is_file: HashMap::new(), children: HashMap::new(), + dir_init: HashMap::new(), }); let result = vfs.read(&file_path).unwrap(); @@ -913,6 +936,7 @@ mod test { files: cache_files, is_file: HashMap::new(), children: HashMap::new(), + dir_init: HashMap::new(), }); let handles: Vec<_> = (0..100) @@ -953,6 +977,7 @@ mod test { files: cache_files, is_file: HashMap::new(), children: HashMap::new(), + dir_init: HashMap::new(), }); for i in 0..50 { diff --git a/src/serve_session.rs b/src/serve_session.rs index 60306ad0..e7956371 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -19,7 +19,7 @@ use crate::{ apply_patch_set, compute_patch_set, AppliedPatchSet, InstanceContext, InstanceSnapshot, PatchSet, RojoTree, }, - snapshot_middleware::{is_script_relevant_path, snapshot_from_vfs}, + snapshot_middleware::{is_script_relevant_path, snapshot_from_vfs, INIT_FILE_PRIORITY}, }; /// Set to `true` to validate on plugin connect (useful for testing, do not enable on production). @@ -155,31 +155,35 @@ fn prefetch_project_files(project: &Project, sync_scripts_only: bool) -> io::Res files: HashMap::new(), is_file: HashMap::new(), children: HashMap::new(), + dir_init: HashMap::new(), }); } let walk_start = Instant::now(); let mut entries: Vec = Vec::new(); - for root in &roots { - if !root.exists() { - continue; - } - for e in WalkDir::new(root).follow_links(true).into_iter().flatten() { + + // Walk the entire project folder so that every file under it + // (nested project files, etc.) is covered by the is_file map. + // P1 returns NotFound for cache misses when prefetch is active. + if folder.exists() { + for e in WalkDir::new(folder) + .follow_links(true) + .into_iter() + .flatten() + { entries.push(e); } } - // Also include the project file itself (read by project middleware). - if let Ok(meta) = std::fs::metadata(&project.file_location) { - if meta.is_file() { - entries.push( - WalkDir::new(&project.file_location) - .into_iter() - .next() - .unwrap() - .unwrap(), - ); + // Also walk $path roots that may be OUTSIDE the project folder + // (e.g. $path: "../module"). These wouldn't be covered above. + for root in &roots { + if !root.exists() || root.starts_with(folder) { + continue; + } + for e in WalkDir::new(root).follow_links(true).into_iter().flatten() { + entries.push(e); } } @@ -215,8 +219,9 @@ fn prefetch_project_files(project: &Project, sync_scripts_only: bool) -> io::Res read_elapsed, ); - let mut is_file_map: HashMap = HashMap::new(); - let mut children_map: HashMap> = HashMap::new(); + let mut is_file_map: HashMap = HashMap::with_capacity(entries.len()); + let mut children_map: HashMap> = + HashMap::with_capacity(dir_count); for entry in &entries { let path = entry.path().to_path_buf(); @@ -242,10 +247,28 @@ fn prefetch_project_files(project: &Project, sync_scripts_only: bool) -> io::Res children_map.len(), ); + let mut dir_init_map: HashMap> = + HashMap::with_capacity(children_map.len()); + for (dir_path, child_paths) in &children_map { + let child_names: std::collections::HashSet<&str> = child_paths + .iter() + .filter_map(|p| p.file_name().and_then(|n| n.to_str())) + .collect(); + let mut found = None; + for &(_, init_name) in INIT_FILE_PRIORITY { + if child_names.contains(init_name) { + found = Some((init_name.to_string(), dir_path.join(init_name))); + break; + } + } + dir_init_map.insert(dir_path.clone(), found); + } + Ok(PrefetchCache { files: file_data.into_iter().collect::>(), is_file: is_file_map, children: children_map, + dir_init: dir_init_map, }) } @@ -273,12 +296,15 @@ impl ServeSession { let prefetch_start = Instant::now(); match prefetch_project_files(&root_project, sync_scripts_only) { Ok(cache) => { + let count = cache.files.len(); log::debug!( "Prefetch total: {} files in {:.1?}", - cache.files.len(), + count, prefetch_start.elapsed() ); - vfs.set_prefetch_cache(cache); + if !cache.is_file.is_empty() { + vfs.set_prefetch_cache(cache); + } } Err(err) => { log::warn!("Prefetch failed, falling back to sequential reads: {err}"); @@ -291,37 +317,45 @@ impl ServeSession { let mut instance_context = InstanceContext::new(); instance_context.sync_scripts_only = sync_scripts_only; - let watch_start = Instant::now(); let mut path_roots = Vec::new(); collect_path_roots( &root_project.tree, root_project.folder_location(), &mut path_roots, ); - let _ = vfs.watch(root_project.folder_location()); - for root in &path_roots { - let _ = vfs.watch(root); - } - log::debug!( - "Set up {} recursive watches in {:.1?}", - path_roots.len() + 1, - watch_start.elapsed(), - ); let snap_start = Instant::now(); log::trace!("Generating snapshot of instances from VFS"); + vfs.set_watch_enabled(false); let snapshot = snapshot_from_vfs(&instance_context, vfs, start_path)?; + vfs.set_watch_enabled(true); log::debug!("Snapshot built in {:.1?}", snap_start.elapsed()); vfs.clear_prefetch_cache(); let patch_start = Instant::now(); - log::trace!("Computing initial patch set"); - let patch_set = compute_patch_set(snapshot, &tree, root_id); + let project_folder = root_project.folder_location().to_path_buf(); + std::thread::scope(|s| { + s.spawn(|| { + let watch_start = Instant::now(); + let _ = vfs.watch(&project_folder); + for root in &path_roots { + let _ = vfs.watch(root); + } + log::debug!( + "Set up {} recursive watches in {:.1?}", + path_roots.len() + 1, + watch_start.elapsed(), + ); + }); - log::trace!("Applying initial patch set"); - apply_patch_set(&mut tree, patch_set); - log::debug!("Patch computed + applied in {:.1?}", patch_start.elapsed()); + log::trace!("Computing initial patch set"); + let patch_set = compute_patch_set(snapshot, &tree, root_id); + + log::trace!("Applying initial patch set"); + apply_patch_set(&mut tree, patch_set); + }); + log::debug!("Watch + patch in {:.1?}", patch_start.elapsed()); Ok((root_project, tree)) } diff --git a/src/snapshot/patch_compute.rs b/src/snapshot/patch_compute.rs index ec585784..1bc10f5b 100644 --- a/src/snapshot/patch_compute.rs +++ b/src/snapshot/patch_compute.rs @@ -274,9 +274,20 @@ fn compute_children_patches( .get_instance(id) .expect("Instance did not exist in tree"); - let instance_children = instance.children().to_vec(); + let instance_children = instance.children(); let snapshot_children = take(&mut snapshot.children); + if instance_children.is_empty() { + for snapshot_child in snapshot_children { + patch_set.added_instances.push(PatchAdd { + parent_id: id, + instance: snapshot_child, + }); + } + return; + } + + let instance_children = instance_children.to_vec(); let match_result = match_forward(snapshot_children, &instance_children, tree, session); // Matched pairs: recursively compute patches. diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 077e7f12..8595c866 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -30,7 +30,6 @@ use serde::{Deserialize, Serialize}; use crate::{ glob::Glob, - project::DEFAULT_PROJECT_NAMES, syncback::{dedup_suffix::strip_dedup_suffix, SyncbackReturn, SyncbackSnapshot}, }; use crate::{ @@ -110,6 +109,27 @@ pub fn snapshot_from_vfs( } } +/// Single source of truth for init-file resolution priority. +/// Project files are checked first, then init files in this order. +/// Used by both `get_dir_middleware` (live) and `prefetch_project_files` (cache). +pub static INIT_FILE_PRIORITY: &[(Middleware, &str)] = &[ + (Middleware::Project, "default.project.json5"), + (Middleware::Project, "default.project.json"), + (Middleware::ModuleScriptDir, "init.luau"), + (Middleware::ServerScriptDir, "init.server.luau"), + (Middleware::ClientScriptDir, "init.client.luau"), + (Middleware::PluginScriptDir, "init.plugin.luau"), + (Middleware::LocalScriptDir, "init.local.luau"), + (Middleware::LegacyScriptDir, "init.legacy.luau"), + (Middleware::CsvDir, "init.csv"), + // Legacy extensions (for backwards compatibility) + // init.server.lua → Script with RunContext.Legacy (old emitLegacyScripts behavior) + // init.client.lua → LocalScript (old emitLegacyScripts behavior) + (Middleware::ModuleScriptDir, "init.lua"), + (Middleware::LegacyScriptDir, "init.server.lua"), + (Middleware::LocalScriptDir, "init.client.lua"), +]; + /// Gets the appropriate middleware for a directory by checking for `init` /// files. This uses an intrinsic priority list and for compatibility, /// that order should be left unchanged. @@ -127,37 +147,24 @@ fn get_dir_middleware<'path>( .ok_or_else(|| anyhow::anyhow!("File name was not valid UTF-8: {}", dir_path.display()))?; let dir_name = strip_dedup_suffix(dir_name); - static INIT_PATHS: OnceLock> = OnceLock::new(); - let order = INIT_PATHS.get_or_init(|| { - vec![ - // Modern extensions (preferred) - (Middleware::ModuleScriptDir, "init.luau"), - (Middleware::ServerScriptDir, "init.server.luau"), - (Middleware::ClientScriptDir, "init.client.luau"), - (Middleware::PluginScriptDir, "init.plugin.luau"), - (Middleware::LocalScriptDir, "init.local.luau"), - (Middleware::LegacyScriptDir, "init.legacy.luau"), - (Middleware::CsvDir, "init.csv"), - // Legacy extensions (for backwards compatibility) - // init.server.lua → Script with RunContext.Legacy (old emitLegacyScripts behavior) - // init.client.lua → LocalScript (old emitLegacyScripts behavior) - (Middleware::ModuleScriptDir, "init.lua"), - (Middleware::LegacyScriptDir, "init.server.lua"), - (Middleware::LocalScriptDir, "init.client.lua"), - ] - }); - - for default_project_name in DEFAULT_PROJECT_NAMES { - let project_path = dir_path.join(default_project_name); - if vfs.metadata(&project_path).with_not_found()?.is_some() { - return Ok((Middleware::Project, dir_name, project_path)); - } + if let Some(cached) = vfs.prefetch_dir_init(dir_path) { + return match cached { + Some((init_name, init_path)) => { + let middleware = INIT_FILE_PRIORITY + .iter() + .find(|(_, name)| *name == init_name) + .map(|(m, _)| *m) + .unwrap_or(Middleware::Dir); + Ok((middleware, dir_name, init_path)) + } + None => Ok((Middleware::Dir, dir_name, dir_path.to_path_buf())), + }; } - for (middleware, name) in order { + for &(middleware, name) in INIT_FILE_PRIORITY { let test_path = dir_path.join(name); if vfs.metadata(&test_path).with_not_found()?.is_some() { - return Ok((*middleware, dir_name, test_path)); + return Ok((middleware, dir_name, test_path)); } } diff --git a/tests/tests/build.rs b/tests/tests/build.rs index 9b61d116..cf01557f 100644 --- a/tests/tests/build.rs +++ b/tests/tests/build.rs @@ -241,19 +241,34 @@ fn parallel_snapshot_with_prefetch_cache() { use std::collections::HashMap; let mut files = HashMap::new(); + let mut is_file = HashMap::new(); + let mut children_map: HashMap> = HashMap::new(); for entry in walkdir::WalkDir::new(root).follow_links(true) { let entry = entry.unwrap(); let path = entry.path().to_path_buf(); + is_file.insert(path.clone(), entry.file_type().is_file()); if entry.file_type().is_file() { files.insert(path.clone(), fs::read(&path).unwrap()); } + if entry.depth() > 0 { + if let Some(parent) = entry.path().parent() { + children_map + .entry(parent.to_path_buf()) + .or_default() + .push(path); + } + } + } + for children in children_map.values_mut() { + children.sort_by(|a, b| a.file_name().cmp(&b.file_name())); } let vfs_cached = memofs::Vfs::new_default(); vfs_cached.set_prefetch_cache(memofs::PrefetchCache { files, - is_file: std::collections::HashMap::new(), - children: std::collections::HashMap::new(), + is_file, + children: children_map, + dir_init: std::collections::HashMap::new(), }); let snap_cached = librojo::snapshot_from_vfs(&ctx, &vfs_cached, &project_path) @@ -359,6 +374,7 @@ fn overlapping_path_roots_no_duplicate_children() { files, is_file, children: children_map, + dir_init: std::collections::HashMap::new(), }); let snap = librojo::snapshot_from_vfs(&ctx, &vfs, &project_path) From aa203177a9adb088659261434120333cbcc57283 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 20:01:15 -0600 Subject: [PATCH 02/12] Enhance VFS functionality and performance improvements - Added VFS recording mode and optimized watch setup to reduce OS file-watch handles during `atlas serve` startup on large projects. - Introduced PrefetchCache directory init-file resolution for faster directory handling during snapshot prefetch. - Fixed propagation of `sync_scripts_only` to `InstanceContext` during tree freshness checks in `ServeSession`. - Enhanced scripts-only mode to filter non-script files at various processing layers. - Set `core.longpaths` git config during repository initialization for improved Windows long-path support. --- CHANGELOG.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8be9b575..0b5dca9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,13 +31,22 @@ Making a new release? Simply add the new header with the version and date undern ## Unreleased -* Add VFS recording mode and non-recursive watch support to reduce OS file-watch handles during `atlas serve` startup on large projects +* Add VFS recording mode and optimized watch setup to reduce OS file-watch handles during `atlas serve` startup on large projects +* Add PrefetchCache directory init-file resolution for faster directory handling during snapshot prefetch * Enhance scripts-only mode to filter non-script files at the snapshot middleware, VFS event, and prefetch layers +* Fix `sync_scripts_only` not being propagated to `InstanceContext` during tree freshness checks in `ServeSession` * Set `core.longpaths` git config during repository initialization for Windows long-path support
Full commit log +- `6168597a` Enhance PrefetchCache with directory init-file resolution +- `14d4bc13` Update instance context in ServeSession for script synchronization +- `0fb5c877` Refactor VFS watching mechanism for improved performance +- `a1b65dbb` Enhance logging for VFS directory watching +- `a1825f7c` Enhance VFS watch functionality to support upgrade from non-recursive to recursive watching +- `9ddd9379` Refine VFS watch behavior to support recursive watching +- `02638535` Add VFS recording mode and enhance git configuration - `2ea1d493` Enhance VFS watch functionality and introduce recording mode - `bfb5815f` Update TODO and enhance git configuration From df773eab711fd843cd092ec6fd5fddee90514941 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 20:05:22 -0600 Subject: [PATCH 03/12] Improve project file prefetching by canonicalizing paths - Updated the `prefetch_project_files` function to canonicalize folder paths before checking if roots start with the folder, preventing false positives with relative paths. - Enhanced the logic to skip non-existent roots early, improving efficiency during the prefetching process. These changes enhance the accuracy and performance of the file prefetching mechanism in the VFS. --- src/serve_session.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/serve_session.rs b/src/serve_session.rs index e7956371..87e844a8 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -178,8 +178,16 @@ fn prefetch_project_files(project: &Project, sync_scripts_only: bool) -> io::Res // Also walk $path roots that may be OUTSIDE the project folder // (e.g. $path: "../module"). These wouldn't be covered above. + // Canonicalize before starts_with to resolve ".." components that + // would otherwise cause false positives (e.g. project/../module + // literally starts_with project/ but is actually a sibling). + let canonical_folder = std::fs::canonicalize(folder).unwrap_or_else(|_| folder.to_path_buf()); for root in &roots { - if !root.exists() || root.starts_with(folder) { + if !root.exists() { + continue; + } + let canonical_root = std::fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); + if canonical_root.starts_with(&canonical_folder) { continue; } for e in WalkDir::new(root).follow_links(true).into_iter().flatten() { From f53e8007ffdb5de94cc8c35154ae75ad0c037597 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 20:38:20 -0600 Subject: [PATCH 04/12] Update TODO.txt to reflect new multi-place clone support - Removed outdated skill section and added a new entry for multi-place clone functionality. - Documented the ability to clone multiple place IDs, enhancing support for games with diverged places. - Suggested dynamic generation of project files to accommodate the new cloning feature. --- TODO.txt | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/TODO.txt b/TODO.txt index 032a41be..0b9fa885 100644 --- a/TODO.txt +++ b/TODO.txt @@ -90,13 +90,9 @@ in all 3 matchers, simply randomize the order of the children - this will break can we allow the terminal to stay so i can like run some git commands myself without having to make a whole new terminal? i also want to just cleanup terminals / use one terminal for syncback and serve... -[skill] -'/playtest-quick' -simply runs the game and checks for literal errors, typically created just after completing a code change -resolve these errors in the project, or if in plan mode create a plan to resolve the errors -this should utilize the mcp to run the game and fetch the errors, make sure the server is fully booted up (INTELLIGENT 5-10s wait) and then fix stop the game - -modify /atlas-push to always trigger /playtest-quick after it finishes pushing, looping back around to fix issues if needed. make sure the errors are related to what we actually pushed, so it doesnt try to fix others work. if it's others work, ask the user if they want to fix it - thought recommend they should do a planning phase. -"/atlas-push no test" or similar it just skips the automated test - -make clone support multi-placeids like we did with defend-67 \ No newline at end of file +[multi place clone] +make clone support multi-placeids like we did with defend-67 in D:\UserGenerated\place-for-defend-67 (i did this manually) +see specifically game/ and game.project.json5, lobby/ and lobby.project.json5. +atlas clone PLACEID1 PLACEID2 ..... +This is extremely helpful for games that have diverged places within the same game, e.g. a lobby and an actual game place +we will probably have to start dynamically generating the x.project.json5 rather than using a template file now though, which is fine.. From 4e9e676cf2cc4d4beb8075884c12488055eefe5c Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 20:39:27 -0600 Subject: [PATCH 05/12] Refactor prefetch_project_files and VFS watch setup for improved efficiency - Updated the `prefetch_project_files` function to perform a shallow walk of the project folder, limiting the depth to enhance performance and avoid unnecessary recursion into specific directories. - Improved the logic for handling $path roots, ensuring correct canonicalization and efficient traversal based on the relationship to the project folder. - Adjusted the VFS watch setup to watch the project folder non-recursively while allowing recursive watching for $path roots, optimizing resource usage during file monitoring. These changes enhance the accuracy and performance of file prefetching and watching in the VFS. --- src/serve_session.rs | 54 +++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/src/serve_session.rs b/src/serve_session.rs index 87e844a8..96650fe9 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -163,12 +163,14 @@ fn prefetch_project_files(project: &Project, sync_scripts_only: bool) -> io::Res let mut entries: Vec = Vec::new(); - // Walk the entire project folder so that every file under it - // (nested project files, etc.) is covered by the is_file map. - // P1 returns NotFound for cache misses when prefetch is active. + // Shallow walk of project folder: just the folder entry + direct + // children. Covers the project file and top-level siblings (nested + // project files). One readdir syscall -- NOT recursive into + // target/, .git/, node_modules/, etc. if folder.exists() { for e in WalkDir::new(folder) .follow_links(true) + .max_depth(1) .into_iter() .flatten() { @@ -176,22 +178,29 @@ fn prefetch_project_files(project: &Project, sync_scripts_only: bool) -> io::Res } } - // Also walk $path roots that may be OUTSIDE the project folder - // (e.g. $path: "../module"). These wouldn't be covered above. - // Canonicalize before starts_with to resolve ".." components that - // would otherwise cause false positives (e.g. project/../module - // literally starts_with project/ but is actually a sibling). + // Recursive walk of each $path root. Canonicalize to resolve ".." + // before the equality/starts_with checks. let canonical_folder = std::fs::canonicalize(folder).unwrap_or_else(|_| folder.to_path_buf()); for root in &roots { if !root.exists() { continue; } let canonical_root = std::fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); - if canonical_root.starts_with(&canonical_folder) { - continue; - } - for e in WalkDir::new(root).follow_links(true).into_iter().flatten() { - entries.push(e); + if canonical_root == canonical_folder { + // $path points to the project folder itself -- shallow walk + // already covers depth 0-1, fill in the subtree. + for e in WalkDir::new(root) + .follow_links(true) + .min_depth(2) + .into_iter() + .flatten() + { + entries.push(e); + } + } else { + for e in WalkDir::new(root).follow_links(true).into_iter().flatten() { + entries.push(e); + } } } @@ -346,12 +355,17 @@ impl ServeSession { std::thread::scope(|s| { s.spawn(|| { let watch_start = Instant::now(); - let _ = vfs.watch(&project_folder); + // Watch each $path root recursively (NOT the project folder -- + // that would walk target/, .git/, etc. for repo-root projects). for root in &path_roots { let _ = vfs.watch(root); } + // Watch project folder non-recursively for project file edits. + vfs.set_watch_recursive(false); + let _ = vfs.watch(&project_folder); + vfs.set_watch_recursive(true); log::debug!( - "Set up {} recursive watches in {:.1?}", + "Set up {} watches in {:.1?}", path_roots.len() + 1, watch_start.elapsed(), ); @@ -396,7 +410,15 @@ impl ServeSession { let ref_path_index = { let mut index = crate::RefPathIndex::new(); let tree_guard = tree.lock().unwrap(); - index.populate_from_dir(root_project.folder_location(), &tree_guard); + let mut ref_roots = Vec::new(); + collect_path_roots( + &root_project.tree, + root_project.folder_location(), + &mut ref_roots, + ); + for root in &ref_roots { + index.populate_from_dir(root, &tree_guard); + } drop(tree_guard); Arc::new(Mutex::new(index)) }; From dc788711c70e3b49397b127b615c196f42e08b0d Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 20:53:28 -0600 Subject: [PATCH 06/12] Fix walked_roots to only include recursively walked directories --- crates/memofs/src/lib.rs | 23 +++++++++++++++++++---- src/serve_session.rs | 6 ++++++ tests/tests/build.rs | 2 ++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/crates/memofs/src/lib.rs b/crates/memofs/src/lib.rs index 48be421f..0e71185f 100644 --- a/crates/memofs/src/lib.rs +++ b/crates/memofs/src/lib.rs @@ -50,6 +50,11 @@ pub struct PrefetchCache { /// Populated during prefetch so that `get_dir_middleware` can skip /// per-file VFS metadata probes entirely. pub dir_init: HashMap>, + /// Directories that were recursively walked during prefetch. A metadata + /// cache miss is only treated as NotFound if the queried path falls + /// under one of these roots. Paths outside walked roots fall through + /// to the backend (they may exist but weren't covered by the walk). + pub walked_roots: Vec, } mod sealed { @@ -288,10 +293,16 @@ impl VfsInner { if let Some(&is_file) = cache.is_file.get(path) { return Ok(Metadata { is_file }); } - return Err(io::Error::new( - io::ErrorKind::NotFound, - "not in prefetch cache", - )); + if cache + .walked_roots + .iter() + .any(|root| path.starts_with(root)) + { + return Err(io::Error::new( + io::ErrorKind::NotFound, + "not in prefetch cache", + )); + } } self.backend.metadata(path) @@ -754,6 +765,7 @@ mod test { is_file: HashMap::new(), children: HashMap::new(), dir_init: HashMap::new(), + walked_roots: Vec::new(), } } @@ -891,6 +903,7 @@ mod test { is_file: HashMap::new(), children: HashMap::new(), dir_init: HashMap::new(), + walked_roots: Vec::new(), }); let result = vfs.read(&file_path).unwrap(); @@ -937,6 +950,7 @@ mod test { is_file: HashMap::new(), children: HashMap::new(), dir_init: HashMap::new(), + walked_roots: Vec::new(), }); let handles: Vec<_> = (0..100) @@ -978,6 +992,7 @@ mod test { is_file: HashMap::new(), children: HashMap::new(), dir_init: HashMap::new(), + walked_roots: Vec::new(), }); for i in 0..50 { diff --git a/src/serve_session.rs b/src/serve_session.rs index 96650fe9..db9daed7 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -156,6 +156,7 @@ fn prefetch_project_files(project: &Project, sync_scripts_only: bool) -> io::Res is_file: HashMap::new(), children: HashMap::new(), dir_init: HashMap::new(), + walked_roots: Vec::new(), }); } @@ -181,11 +182,15 @@ fn prefetch_project_files(project: &Project, sync_scripts_only: bool) -> io::Res // Recursive walk of each $path root. Canonicalize to resolve ".." // before the equality/starts_with checks. let canonical_folder = std::fs::canonicalize(folder).unwrap_or_else(|_| folder.to_path_buf()); + let mut walked_roots: Vec = Vec::new(); for root in &roots { if !root.exists() { continue; } let canonical_root = std::fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); + if canonical_root != canonical_folder { + walked_roots.push(canonical_root.clone()); + } if canonical_root == canonical_folder { // $path points to the project folder itself -- shallow walk // already covers depth 0-1, fill in the subtree. @@ -286,6 +291,7 @@ fn prefetch_project_files(project: &Project, sync_scripts_only: bool) -> io::Res is_file: is_file_map, children: children_map, dir_init: dir_init_map, + walked_roots, }) } diff --git a/tests/tests/build.rs b/tests/tests/build.rs index cf01557f..2d8865b3 100644 --- a/tests/tests/build.rs +++ b/tests/tests/build.rs @@ -269,6 +269,7 @@ fn parallel_snapshot_with_prefetch_cache() { is_file, children: children_map, dir_init: std::collections::HashMap::new(), + walked_roots: Vec::new(), }); let snap_cached = librojo::snapshot_from_vfs(&ctx, &vfs_cached, &project_path) @@ -375,6 +376,7 @@ fn overlapping_path_roots_no_duplicate_children() { is_file, children: children_map, dir_init: std::collections::HashMap::new(), + walked_roots: Vec::new(), }); let snap = librojo::snapshot_from_vfs(&ctx, &vfs, &project_path) From 5ff0ed3f99fcafed26536c52f93facae3c887c84 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 20:55:28 -0600 Subject: [PATCH 07/12] Refactor cache lookup logic in VFS to streamline path checks - Simplified the condition for checking if a path is within the walked roots by removing unnecessary variable assignments. - This change enhances the clarity and efficiency of the metadata retrieval process in the VFS. --- crates/memofs/src/lib.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/memofs/src/lib.rs b/crates/memofs/src/lib.rs index 0e71185f..62c72557 100644 --- a/crates/memofs/src/lib.rs +++ b/crates/memofs/src/lib.rs @@ -293,11 +293,7 @@ impl VfsInner { if let Some(&is_file) = cache.is_file.get(path) { return Ok(Metadata { is_file }); } - if cache - .walked_roots - .iter() - .any(|root| path.starts_with(root)) - { + if cache.walked_roots.iter().any(|root| path.starts_with(root)) { return Err(io::Error::new( io::ErrorKind::NotFound, "not in prefetch cache", From e86ebdadbc9319cd5ea36817dcee6c7a0a80b6ee Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 21:07:35 -0600 Subject: [PATCH 08/12] Fix walked_roots to store raw paths, not canonical --- src/serve_session.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/serve_session.rs b/src/serve_session.rs index db9daed7..010580bc 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -189,7 +189,7 @@ fn prefetch_project_files(project: &Project, sync_scripts_only: bool) -> io::Res } let canonical_root = std::fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); if canonical_root != canonical_folder { - walked_roots.push(canonical_root.clone()); + walked_roots.push(root.clone()); } if canonical_root == canonical_folder { // $path points to the project folder itself -- shallow walk From 8b18dc451af06542e7eaf780171304a53ad9e807 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 21:23:14 -0600 Subject: [PATCH 09/12] Add guard test for INIT_FILE_PRIORITY vs DEFAULT_PROJECT_NAMES sync --- src/snapshot_middleware/mod.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 8595c866..29b880e8 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -112,6 +112,10 @@ pub fn snapshot_from_vfs( /// Single source of truth for init-file resolution priority. /// Project files are checked first, then init files in this order. /// Used by both `get_dir_middleware` (live) and `prefetch_project_files` (cache). +/// +/// NOTE: The project file entries must match `DEFAULT_PROJECT_NAMES` in +/// `project.rs`. The `init_file_priority_includes_all_project_names` test +/// guards against drift. pub static INIT_FILE_PRIORITY: &[(Middleware, &str)] = &[ (Middleware::Project, "default.project.json5"), (Middleware::Project, "default.project.json"), @@ -526,6 +530,22 @@ mod test { use memofs::{InMemoryFs, VfsSnapshot}; + #[test] + fn init_file_priority_includes_all_project_names() { + use crate::project::DEFAULT_PROJECT_NAMES; + let priority_names: Vec<&str> = INIT_FILE_PRIORITY + .iter() + .filter(|(m, _)| *m == Middleware::Project) + .map(|(_, name)| *name) + .collect(); + for project_name in DEFAULT_PROJECT_NAMES { + assert!( + priority_names.contains(&project_name), + "DEFAULT_PROJECT_NAMES entry {project_name:?} missing from INIT_FILE_PRIORITY" + ); + } + } + #[test] fn is_script_covers_all_script_types() { assert!(Middleware::ServerScript.is_script()); From 5275d45adce40060f1494cd4ee98af91c43fe3da Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 21:51:22 -0600 Subject: [PATCH 10/12] Add is_watch_enabled method to VFS and enhance prefetch cache tests - Introduced `is_watch_enabled` method to check the status of automatic file watching in the VFS. - Added a new test for prefetch cache to validate metadata retrieval and error handling for walked roots and external paths. - Updated the `watch` method in `StdBackend` for cleaner logic. - Enhanced the `dir_init` field in `PrefetchCache` to store resolved init-file information for directories, improving prefetch operations. --- crates/memofs/src/lib.rs | 62 ++++++++++++++++++++++++++++++++ crates/memofs/src/std_backend.rs | 8 ++--- src/lib.rs | 2 +- src/serve_session.rs | 3 +- src/snapshot_middleware/mod.rs | 6 ++++ tests/tests/build.rs | 19 +++++++++- 6 files changed, 91 insertions(+), 9 deletions(-) diff --git a/crates/memofs/src/lib.rs b/crates/memofs/src/lib.rs index 62c72557..c99acb1a 100644 --- a/crates/memofs/src/lib.rs +++ b/crates/memofs/src/lib.rs @@ -419,6 +419,11 @@ impl Vfs { } } + /// Returns whether automatic file watching is currently enabled. + pub fn is_watch_enabled(&self) -> bool { + self.inner.lock().unwrap().watch_enabled + } + /// Turns automatic file watching on or off. Enabled by default. /// /// Turning off file watching may be useful for single-use cases, especially @@ -1008,6 +1013,63 @@ mod test { } } + #[test] + fn prefetch_cache_walked_roots_fast_reject() { + let mut imfs = InMemoryFs::new(); + imfs.load_snapshot( + "/root", + VfsSnapshot::dir(HashMap::from([ + ("known.txt", VfsSnapshot::file("hello")), + ( + "subdir", + VfsSnapshot::dir(HashMap::from([("nested.txt", VfsSnapshot::file("world"))])), + ), + ])), + ) + .unwrap(); + imfs.load_snapshot("/outside/other.txt", VfsSnapshot::file("external")) + .unwrap(); + + let vfs = Vfs::new(imfs); + let mut is_file = HashMap::new(); + is_file.insert(PathBuf::from("/root"), false); + is_file.insert(PathBuf::from("/root/known.txt"), true); + is_file.insert(PathBuf::from("/root/subdir"), false); + + vfs.set_prefetch_cache(PrefetchCache { + files: HashMap::new(), + is_file, + children: HashMap::new(), + dir_init: HashMap::new(), + walked_roots: vec![PathBuf::from("/root")], + }); + + let known = vfs.metadata("/root/known.txt").unwrap(); + assert!( + known.is_file(), + "cached entry should return correct metadata" + ); + + let subdir = vfs.metadata("/root/subdir").unwrap(); + assert!( + subdir.is_dir(), + "cached directory should return correct metadata" + ); + + let missing = vfs.metadata("/root/subdir/nested.txt"); + assert!( + missing.is_err(), + "path under walked root but not in cache should return NotFound" + ); + assert_eq!(missing.unwrap_err().kind(), io::ErrorKind::NotFound); + + let outside = vfs.metadata("/outside/other.txt").unwrap(); + assert!( + outside.is_file(), + "path outside walked roots should fall through to backend" + ); + } + #[test] fn prefetch_cache_read_after_write_ignores_cache() { let mut imfs = InMemoryFs::new(); diff --git a/crates/memofs/src/std_backend.rs b/crates/memofs/src/std_backend.rs index f3c2c145..bf44ca28 100644 --- a/crates/memofs/src/std_backend.rs +++ b/crates/memofs/src/std_backend.rs @@ -331,12 +331,8 @@ impl VfsBackend for StdBackend { } fn watch(&mut self, path: &Path, recursive: bool) -> io::Result<()> { - if self.watches.contains(path) { - if !recursive || self.recursive_watches.contains(path) { - return Ok(()); - } - // Upgrade: path was watched non-recursively, now needs recursive. - // Fall through to re-issue the watch with RecursiveMode::Recursive. + if self.watches.contains(path) && (!recursive || self.recursive_watches.contains(path)) { + return Ok(()); } if path .ancestors() diff --git a/src/lib.rs b/src/lib.rs index c7b4f501..1269c257 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,6 +39,6 @@ pub use snapshot::{ InstanceContext, InstanceMetadata, InstanceSnapshot, InstanceWithMeta, InstanceWithMetaMut, RojoDescendants, RojoTree, }; -pub use snapshot_middleware::{snapshot_from_vfs, Middleware, ScriptType}; +pub use snapshot_middleware::{snapshot_from_vfs, Middleware, ScriptType, INIT_FILE_PRIORITY}; pub use syncback::{syncback_loop, FsSnapshot, SyncbackData, SyncbackResult, SyncbackSnapshot}; pub use web::interface as web_api; diff --git a/src/serve_session.rs b/src/serve_session.rs index 010580bc..f71f756f 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -349,9 +349,10 @@ impl ServeSession { let snap_start = Instant::now(); log::trace!("Generating snapshot of instances from VFS"); + let was_watch_enabled = vfs.is_watch_enabled(); vfs.set_watch_enabled(false); let snapshot = snapshot_from_vfs(&instance_context, vfs, start_path)?; - vfs.set_watch_enabled(true); + vfs.set_watch_enabled(was_watch_enabled); log::debug!("Snapshot built in {:.1?}", snap_start.elapsed()); vfs.clear_prefetch_cache(); diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 29b880e8..c1273f73 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -544,6 +544,12 @@ mod test { "DEFAULT_PROJECT_NAMES entry {project_name:?} missing from INIT_FILE_PRIORITY" ); } + for &name in &priority_names { + assert!( + DEFAULT_PROJECT_NAMES.contains(&name), + "INIT_FILE_PRIORITY has Project entry {name:?} not in DEFAULT_PROJECT_NAMES" + ); + } } #[test] diff --git a/tests/tests/build.rs b/tests/tests/build.rs index 2d8865b3..69e6e0b4 100644 --- a/tests/tests/build.rs +++ b/tests/tests/build.rs @@ -263,12 +263,29 @@ fn parallel_snapshot_with_prefetch_cache() { children.sort_by(|a, b| a.file_name().cmp(&b.file_name())); } + let mut dir_init: HashMap> = + HashMap::new(); + for (dir_path, child_paths) in &children_map { + let child_names: std::collections::HashSet<&str> = child_paths + .iter() + .filter_map(|p| p.file_name().and_then(|n| n.to_str())) + .collect(); + let mut found = None; + for &(_, init_name) in librojo::INIT_FILE_PRIORITY { + if child_names.contains(init_name) { + found = Some((init_name.to_string(), dir_path.join(init_name))); + break; + } + } + dir_init.insert(dir_path.clone(), found); + } + let vfs_cached = memofs::Vfs::new_default(); vfs_cached.set_prefetch_cache(memofs::PrefetchCache { files, is_file, children: children_map, - dir_init: std::collections::HashMap::new(), + dir_init, walked_roots: Vec::new(), }); From 04565dab91b5dd94432b26e354e16cc31053b6a5 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 22:01:12 -0600 Subject: [PATCH 11/12] Update TODO.txt to include syncback optimization task - Added a new entry under [syncback] to optimize the syncback process, referencing previous improvements made to the serve functionality. - This update aims to enhance the efficiency of the syncback feature in line with ongoing project enhancements. --- TODO.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TODO.txt b/TODO.txt index 0b9fa885..ca920975 100644 --- a/TODO.txt +++ b/TODO.txt @@ -96,3 +96,6 @@ see specifically game/ and game.project.json5, lobby/ and lobby.project.json5. atlas clone PLACEID1 PLACEID2 ..... This is extremely helpful for games that have diverged places within the same game, e.g. a lobby and an actual game place we will probably have to start dynamically generating the x.project.json5 rather than using a template file now though, which is fine.. + +[syncback] +optimize syncback like we did serve. From e8daa9e9a50f3422155de43545d8b956f31158ae Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Mar 2026 22:02:25 -0600 Subject: [PATCH 12/12] Enhance VFS and PrefetchCache functionality - Optimized project file prefetching with shallow folder walk and depth-limited traversal for `$path` roots, improving efficiency. - Implemented non-recursive watching for project folders while maintaining recursive watches for `$path` roots, reducing watch handle count. - Added `is_watch_enabled` method to VFS for querying file watching status. - Fixed issues with prefetch path canonicalization and `walked_roots` tracking to ensure accurate path storage. - Introduced guard tests for `INIT_FILE_PRIORITY` and `DEFAULT_PROJECT_NAMES` synchronization, along with prefetch cache integration tests for metadata retrieval. --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b5dca9a..cd457315 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,13 +33,28 @@ Making a new release? Simply add the new header with the version and date undern * Add VFS recording mode and optimized watch setup to reduce OS file-watch handles during `atlas serve` startup on large projects * Add PrefetchCache directory init-file resolution for faster directory handling during snapshot prefetch +* Optimize project file prefetching with shallow folder walk and depth-limited traversal for `$path` roots +* Watch project folder non-recursively while using recursive watches only for `$path` roots, reducing watch handle count +* Add `is_watch_enabled` method to VFS for querying file watching status * Enhance scripts-only mode to filter non-script files at the snapshot middleware, VFS event, and prefetch layers * Fix `sync_scripts_only` not being propagated to `InstanceContext` during tree freshness checks in `ServeSession` +* Fix prefetch path canonicalization to avoid false positives with relative paths +* Fix `walked_roots` tracking to correctly store raw paths and only include recursively watched directories * Set `core.longpaths` git config during repository initialization for Windows long-path support +* Add guard test ensuring `INIT_FILE_PRIORITY` and `DEFAULT_PROJECT_NAMES` arrays stay in sync +* Add prefetch cache integration tests for metadata retrieval and walked-root boundaries
Full commit log +- `5275d45a` Add is_watch_enabled method to VFS and enhance prefetch cache tests +- `8b18dc45` Add guard test for INIT_FILE_PRIORITY vs DEFAULT_PROJECT_NAMES sync +- `e86ebdad` Fix walked_roots to store raw paths, not canonical +- `5ff0ed3f` Refactor cache lookup logic in VFS to streamline path checks +- `dc788711` Fix walked_roots to only include recursively walked directories +- `4e9e676c` Refactor prefetch_project_files and VFS watch setup for improved efficiency +- `df773eab` Improve project file prefetching by canonicalizing paths +- `aa203177` Enhance VFS functionality and performance improvements - `6168597a` Enhance PrefetchCache with directory init-file resolution - `14d4bc13` Update instance context in ServeSession for script synchronization - `0fb5c877` Refactor VFS watching mechanism for improved performance