diff --git a/crates/memofs/src/lib.rs b/crates/memofs/src/lib.rs index e4bd33ec..966534a2 100644 --- a/crates/memofs/src/lib.rs +++ b/crates/memofs/src/lib.rs @@ -32,7 +32,7 @@ pub use noop_backend::NoopBackend; pub use snapshot::VfsSnapshot; pub use std_backend::{CriticalErrorHandler, StdBackend, WatcherCriticalError}; -/// Pre-read file contents and canonical paths for fast startup. +/// Pre-read file contents, canonical paths, and metadata for fast startup. /// /// Populated in parallel (via rayon + walkdir) before snapshot building, /// then consumed by VFS reads during `snapshot_from_vfs()`. Cleared after @@ -40,6 +40,9 @@ pub use std_backend::{CriticalErrorHandler, StdBackend, WatcherCriticalError}; pub struct PrefetchCache { pub files: HashMap>, pub canonical: HashMap, + /// `true` = file, `false` = directory. Paths not in the map fall through + /// to the backend (e.g. init-file probes for paths that don't exist). + pub is_file: HashMap, } mod sealed { @@ -246,6 +249,13 @@ impl VfsInner { fn metadata>(&mut self, path: P) -> io::Result { let path = path.as_ref(); + + if let Some(cache) = &self.prefetch_cache { + if let Some(&is_file) = cache.is_file.get(path) { + return Ok(Metadata { is_file }); + } + } + self.backend.metadata(path) } @@ -754,6 +764,7 @@ mod test { .into_iter() .map(|(k, v)| (PathBuf::from(k), PathBuf::from(v))) .collect(), + is_file: HashMap::new(), } } @@ -871,9 +882,162 @@ mod test { vfs.set_prefetch_cache(PrefetchCache { files: cache_files, canonical: HashMap::new(), + is_file: HashMap::new(), }); let result = vfs.read(&file_path).unwrap(); assert_eq!(result.as_slice(), contents.as_bytes()); } + + #[test] + fn prefetch_cache_read_to_string_invalid_utf8() { + let imfs = InMemoryFs::new(); + let vfs = Vfs::new(imfs); + vfs.set_prefetch_cache(make_prefetch( + vec![("test", &[0xFF, 0xFE, 0x00, 0x80])], + vec![], + )); + + let err = vfs.read_to_string("test").unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + } + + #[test] + fn prefetch_cache_canonicalize_depletion() { + let mut imfs = InMemoryFs::new(); + imfs.load_snapshot("/real/path", VfsSnapshot::file("x")) + .unwrap(); + let vfs = Vfs::new(imfs); + vfs.set_prefetch_cache(make_prefetch( + vec![], + vec![("/real/path", "/canonical/path")], + )); + + assert_eq!( + vfs.canonicalize("/real/path").unwrap(), + PathBuf::from("/canonical/path"), + "First call should hit cache" + ); + assert_eq!( + vfs.canonicalize("/real/path").unwrap(), + PathBuf::from("/real/path"), + "Second call should fall through to InMemoryFs backend" + ); + } + + #[test] + fn prefetch_cache_set_overwrite_previous() { + let imfs = InMemoryFs::new(); + let vfs = Vfs::new(imfs); + + vfs.set_prefetch_cache(make_prefetch(vec![("a", b"first")], vec![])); + vfs.set_prefetch_cache(make_prefetch(vec![("a", b"second")], vec![])); + + assert_eq!(vfs.read("a").unwrap().as_slice(), b"second"); + } + + #[test] + fn prefetch_cache_concurrent_reads_from_threads() { + use std::sync::Arc; + + let dir = tempfile::tempdir().unwrap(); + let mut cache_files = HashMap::new(); + for i in 0..100 { + let path = dir.path().join(format!("file_{i}.txt")); + let content = format!("content_{i}"); + fs_err::write(&path, &content).unwrap(); + cache_files.insert(path, content.into_bytes()); + } + + let vfs = Arc::new(Vfs::new(StdBackend::new_for_testing())); + vfs.set_prefetch_cache(PrefetchCache { + files: cache_files, + canonical: HashMap::new(), + is_file: HashMap::new(), + }); + + let handles: Vec<_> = (0..100) + .map(|i| { + let vfs = Arc::clone(&vfs); + let path = dir.path().join(format!("file_{i}.txt")); + let expected = format!("content_{i}"); + std::thread::spawn(move || { + let data = vfs.read(&path).unwrap(); + assert_eq!( + String::from_utf8(data.to_vec()).unwrap(), + expected, + "File {i} content mismatch" + ); + }) + }) + .collect(); + + for h in handles { + h.join().expect("Thread panicked"); + } + } + + #[test] + fn prefetch_cache_many_files_std_backend() { + let dir = tempfile::tempdir().unwrap(); + let mut cache_files = HashMap::new(); + let mut canonical = HashMap::new(); + + for i in 0..50 { + let path = dir.path().join(format!("f{i}.txt")); + let content = format!("data_{i}"); + fs_err::write(&path, &content).unwrap(); + cache_files.insert(path.clone(), content.into_bytes()); + if let Ok(c) = path.canonicalize() { + canonical.insert(path, c); + } + } + + let vfs = Vfs::new(StdBackend::new_for_testing()); + vfs.set_prefetch_cache(PrefetchCache { + files: cache_files, + canonical, + is_file: HashMap::new(), + }); + + for i in 0..50 { + let path = dir.path().join(format!("f{i}.txt")); + let data = vfs.read_to_string(&path).unwrap(); + assert_eq!(data.as_str(), &format!("data_{i}")); + } + + for i in 0..50 { + let path = dir.path().join(format!("f{i}.txt")); + let data = vfs.read_to_string(&path).unwrap(); + assert_eq!( + data.as_str(), + &format!("data_{i}"), + "Second read (backend) of f{i}.txt diverged" + ); + } + } + + #[test] + fn prefetch_cache_read_after_write_ignores_cache() { + let mut imfs = InMemoryFs::new(); + imfs.load_snapshot("test", VfsSnapshot::file("original")) + .unwrap(); + let vfs = Vfs::new(imfs); + vfs.set_prefetch_cache(make_prefetch(vec![("test", b"cached")], vec![])); + + vfs.write("test", b"written").unwrap(); + let data = vfs.read("test").unwrap(); + assert_eq!( + data.as_slice(), + b"cached", + "Cache entry should still be consumed first" + ); + + let data2 = vfs.read("test").unwrap(); + assert_eq!( + data2.as_slice(), + b"written", + "After cache depleted, should see the written data" + ); + } } diff --git a/src/cli/init.rs b/src/cli/init.rs index f322a9ce..d37a6059 100644 --- a/src/cli/init.rs +++ b/src/cli/init.rs @@ -131,10 +131,10 @@ impl InitCommand { ("core.autocrlf", "false"), ("core.eol", "lf"), ("core.safecrlf", "false"), - ("core.splitIndex", "true"), + // ("core.splitIndex", "true"), // ("core.fsmonitor", "true"), - ("core.untrackedcache", "true"), - ("feature.manyFiles", "true"), + // ("core.untrackedcache", "true"), + // ("feature.manyFiles", "true"), ] { let _ = Command::new("git") .args(["config", "--local", key, value]) diff --git a/src/rojo_ref.rs b/src/rojo_ref.rs index da185c7d..6e923fa6 100644 --- a/src/rojo_ref.rs +++ b/src/rojo_ref.rs @@ -385,8 +385,7 @@ impl RefPathIndex { .into_iter() .filter_map(|e: Result| e.ok()) .filter(|e: &walkdir::DirEntry| { - e.file_type().is_file() - && e.file_name().to_str().is_some_and(is_meta_or_model) + e.file_type().is_file() && e.file_name().to_str().is_some_and(is_meta_or_model) }) .map(|e: walkdir::DirEntry| e.into_path()) .collect(); diff --git a/src/serve_session.rs b/src/serve_session.rs index 58edfb51..6c6f3d78 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -2,7 +2,7 @@ use std::{ collections::HashSet, io, net::IpAddr, - path::Path, + path::{Path, PathBuf}, sync::{Arc, Mutex, MutexGuard}, time::Instant, }; @@ -125,19 +125,66 @@ pub struct ServeSession { git_repo_root: Option, } -/// Walk a directory tree in parallel, reading all file contents and computing -/// canonical paths. Returns a [`PrefetchCache`] that can be loaded into the -/// VFS to eliminate per-file I/O during the initial snapshot build. -fn prefetch_project_files(root: &Path) -> io::Result { +/// Collect all filesystem paths reachable from the project tree's `$path` +/// entries, then read file contents and compute canonical paths in parallel. +fn prefetch_project_files(project: &Project) -> io::Result { use rayon::prelude::*; use std::collections::HashMap; use walkdir::WalkDir; - let entries: Vec = WalkDir::new(root) - .follow_links(true) - .into_iter() - .filter_map(|e| e.ok()) - .collect(); + let folder = project.folder_location(); + + let mut roots: Vec = Vec::new(); + collect_path_roots(&project.tree, folder, &mut roots); + + if roots.is_empty() { + return Ok(PrefetchCache { + files: HashMap::new(), + canonical: HashMap::new(), + is_file: HashMap::new(), + }); + } + + let walk_start = Instant::now(); + + let mut entries: Vec = Vec::new(); + for root in &roots { + if !root.exists() { + continue; + } + for entry in WalkDir::new(root).follow_links(true) { + if let Ok(e) = entry { + entries.push(e); + } + } + } + + // Also include the project file itself (read by project middleware). + if let Ok(meta) = std::fs::metadata(&project.file_location) { + if meta.is_file() { + entries.push( + WalkDir::new(&project.file_location) + .into_iter() + .next() + .unwrap() + .unwrap(), + ); + } + } + + let walk_elapsed = walk_start.elapsed(); + let file_count = entries.iter().filter(|e| e.file_type().is_file()).count(); + let dir_count = entries.len() - file_count; + + log::info!( + "Prefetch walk: {} files + {} dirs from {} root(s) in {:.1?}", + file_count, + dir_count, + roots.len(), + walk_elapsed, + ); + + let read_start = Instant::now(); let file_data: Vec<_> = entries .par_iter() @@ -148,6 +195,9 @@ fn prefetch_project_files(root: &Path) -> io::Result { }) .collect(); + let read_elapsed = read_start.elapsed(); + let canon_start = Instant::now(); + let canonical_data: Vec<_> = entries .par_iter() .filter_map(|e| { @@ -156,12 +206,39 @@ fn prefetch_project_files(root: &Path) -> io::Result { }) .collect(); + let canon_elapsed = canon_start.elapsed(); + + log::info!( + "Prefetch I/O: read {} files in {:.1?}, canonicalize {} paths in {:.1?}", + file_data.len(), + read_elapsed, + canonical_data.len(), + canon_elapsed, + ); + + let is_file: HashMap<_, _> = entries + .iter() + .map(|e| (e.path().to_path_buf(), e.file_type().is_file())) + .collect(); + Ok(PrefetchCache { files: file_data.into_iter().collect::>(), canonical: canonical_data.into_iter().collect::>(), + is_file, }) } +/// Recursively collect all `$path` directories from the project tree. +fn collect_path_roots(node: &crate::project::ProjectNode, base: &Path, out: &mut Vec) { + if let Some(path_node) = &node.path { + let resolved = base.join(path_node.path()); + out.push(resolved); + } + for child_node in node.children.values() { + collect_path_roots(child_node, base, out); + } +} + impl ServeSession { /// Shared initialization: loads the project and builds the initial /// snapshot tree. Used by both `new()` and `new_oneshot()`. @@ -172,10 +249,10 @@ impl ServeSession { if std::env::var("ATLAS_SEQUENTIAL").is_err() { let prefetch_start = Instant::now(); - match prefetch_project_files(root_project.folder_location()) { + match prefetch_project_files(&root_project) { Ok(cache) => { log::info!( - "Prefetched {} files + {} canonical paths in {:.1?}", + "Prefetch total: {} files + {} canonical paths in {:.1?}", cache.files.len(), cache.canonical.len(), prefetch_start.elapsed() @@ -192,16 +269,20 @@ impl ServeSession { let root_id = tree.get_root_id(); let instance_context = InstanceContext::new(); + let snap_start = Instant::now(); log::trace!("Generating snapshot of instances from VFS"); let snapshot = snapshot_from_vfs(&instance_context, vfs, start_path)?; + log::info!("Snapshot built in {:.1?}", snap_start.elapsed()); vfs.clear_prefetch_cache(); + let patch_start = Instant::now(); log::trace!("Computing initial patch set"); let patch_set = compute_patch_set(snapshot, &tree, root_id); log::trace!("Applying initial patch set"); apply_patch_set(&mut tree, patch_set); + log::info!("Patch computed + applied in {:.1?}", patch_start.elapsed()); Ok((root_project, tree)) } diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index acda7250..1c5e92df 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -362,6 +362,174 @@ mod test { use memofs::{InMemoryFs, VfsSnapshot}; + #[test] + fn parallel_wide_directory_ordering() { + let children: Vec<_> = (0..50) + .map(|i| { + ( + format!("child_{i:03}.luau"), + VfsSnapshot::file(format!("-- {i}")), + ) + }) + .collect(); + + let mut imfs = InMemoryFs::new(); + imfs.load_snapshot("/wide", VfsSnapshot::dir(children)) + .unwrap(); + let vfs = Vfs::new(imfs); + + let snap = snapshot_dir( + &InstanceContext::default(), + &vfs, + Path::new("/wide"), + "wide", + ) + .unwrap() + .unwrap(); + + let names: Vec<&str> = snap.children.iter().map(|c| c.name.as_ref()).collect(); + let mut sorted = names.clone(); + sorted.sort(); + assert_eq!( + names, sorted, + "Children should be in sorted (read_dir) order" + ); + assert_eq!(snap.children.len(), 50); + } + + #[test] + fn parallel_wide_directory_deterministic_across_runs() { + fn make_children() -> Vec<(String, VfsSnapshot)> { + (0..30) + .map(|i| { + ( + format!("mod_{i:03}.luau"), + VfsSnapshot::file(format!("-- {i}")), + ) + }) + .collect() + } + + fn names_from_fresh() -> Vec { + let mut imfs = InMemoryFs::new(); + imfs.load_snapshot("/root", VfsSnapshot::dir(make_children())) + .unwrap(); + let vfs = Vfs::new(imfs); + let snap = snapshot_dir( + &InstanceContext::default(), + &vfs, + Path::new("/root"), + "root", + ) + .unwrap() + .unwrap(); + snap.children.iter().map(|c| c.name.to_string()).collect() + } + + let baseline = names_from_fresh(); + + for i in 0..20 { + assert_eq!( + baseline, + names_from_fresh(), + "Child order diverged on iteration {i}" + ); + } + } + + #[test] + fn parallel_deep_wide_tree() { + fn make_tree(depth: usize, width: usize) -> VfsSnapshot { + if depth == 0 { + return VfsSnapshot::file("-- leaf"); + } + let children: Vec<_> = (0..width) + .map(|i| { + if i % 3 == 0 { + (format!("dir_{i}"), make_tree(depth - 1, width)) + } else { + ( + format!("file_{i}.luau"), + VfsSnapshot::file(format!("-- d{depth} f{i}")), + ) + } + }) + .collect(); + VfsSnapshot::dir(children) + } + + let mut imfs = InMemoryFs::new(); + imfs.load_snapshot("/deep", make_tree(4, 8)).unwrap(); + let vfs = Vfs::new(imfs); + + let snap = snapshot_dir( + &InstanceContext::default(), + &vfs, + Path::new("/deep"), + "deep", + ) + .unwrap() + .unwrap(); + + fn count_nodes(snap: &InstanceSnapshot) -> usize { + 1 + snap.children.iter().map(|c| count_nodes(c)).sum::() + } + let total = count_nodes(&snap); + assert!(total > 100, "Should have many nodes, got {total}"); + } + + #[test] + fn parallel_mixed_middleware_types() { + let mut imfs = InMemoryFs::new(); + imfs.load_snapshot( + "/mixed", + VfsSnapshot::dir([ + ("script.server.luau", VfsSnapshot::file("-- server")), + ("module.luau", VfsSnapshot::file("return {}")), + ("data.json", VfsSnapshot::file(r#"{"key": "value"}"#)), + ("notes.txt", VfsSnapshot::file("hello")), + ( + "sub", + VfsSnapshot::dir([ + ("inner.luau", VfsSnapshot::file("return 1")), + ( + "deep", + VfsSnapshot::dir([("leaf.luau", VfsSnapshot::file("return 2"))]), + ), + ]), + ), + ("another.luau", VfsSnapshot::file("return 3")), + ( + "model.model.json", + VfsSnapshot::file(r#"{"className": "Part", "properties": {}}"#), + ), + ]), + ) + .unwrap(); + let vfs = Vfs::new(imfs); + + let snap = snapshot_dir( + &InstanceContext::default(), + &vfs, + Path::new("/mixed"), + "mixed", + ) + .unwrap() + .unwrap(); + + let names: Vec<&str> = snap.children.iter().map(|c| c.name.as_ref()).collect(); + assert!(names.contains(&"script"), "Missing server script"); + assert!(names.contains(&"module"), "Missing module script"); + assert!(names.contains(&"data"), "Missing json data"); + assert!(names.contains(&"notes"), "Missing txt"); + assert!(names.contains(&"sub"), "Missing subdirectory"); + assert!(names.contains(&"another"), "Missing another script"); + assert!(names.contains(&"model"), "Missing model"); + + let sub = snap.children.iter().find(|c| c.name == "sub").unwrap(); + assert_eq!(sub.children.len(), 2, "sub should have inner + deep"); + } + #[test] fn empty_folder() { let mut imfs = InMemoryFs::new(); diff --git a/tests/tests/build.rs b/tests/tests/build.rs index 58792c20..7568421e 100644 --- a/tests/tests/build.rs +++ b/tests/tests/build.rs @@ -114,6 +114,20 @@ fn run_build_test(test_name: &str) { }); } +fn snapshot_debug(snap: &librojo::InstanceSnapshot) -> String { + fn recurse(snap: &librojo::InstanceSnapshot, depth: usize, out: &mut String) { + use std::fmt::Write; + let indent = " ".repeat(depth); + writeln!(out, "{}{} [{}]", indent, snap.name, snap.class_name).unwrap(); + for child in &snap.children { + recurse(child, depth + 1, out); + } + } + let mut s = String::new(); + recurse(snap, 0, &mut s); + s +} + #[test] fn parallel_snapshot_determinism() { use crate::rojo_test::io_util::SERVE_TESTS_PATH; @@ -124,20 +138,6 @@ fn parallel_snapshot_determinism() { let vfs = memofs::Vfs::new_default(); let ctx = librojo::InstanceContext::default(); - fn snapshot_debug(snap: &librojo::InstanceSnapshot) -> String { - fn recurse(snap: &librojo::InstanceSnapshot, depth: usize, out: &mut String) { - use std::fmt::Write; - let indent = " ".repeat(depth); - writeln!(out, "{}{} [{}]", indent, snap.name, snap.class_name).unwrap(); - for child in &snap.children { - recurse(child, depth + 1, out); - } - } - let mut s = String::new(); - recurse(snap, 0, &mut s); - s - } - let first = librojo::snapshot_from_vfs(&ctx, &vfs, &fixture_path) .expect("snapshot failed") .expect("snapshot returned None"); @@ -155,3 +155,149 @@ fn parallel_snapshot_determinism() { ); } } + +#[test] +fn parallel_snapshot_determinism_stress() { + let _ = env_logger::try_init(); + + let dir = tempdir().expect("couldn't create temp dir"); + let root = dir.path(); + let src = root.join("src"); + fs::create_dir(&src).unwrap(); + + for i in 0..40 { + fs::write(src.join(format!("mod_{i:03}.luau")), format!("return {i}")).unwrap(); + } + for i in 0..10 { + let sub = src.join(format!("pkg_{i:02}")); + fs::create_dir(&sub).unwrap(); + for j in 0..5 { + fs::write(sub.join(format!("sub_{j}.luau")), format!("return {j}")).unwrap(); + } + } + + fs::write( + root.join("default.project.json5"), + r#"{ "name": "StressTest", "tree": { "$path": "src" } }"#, + ) + .unwrap(); + + let ctx = librojo::InstanceContext::default(); + let project_path = root.join("default.project.json5"); + + let vfs = memofs::Vfs::new_default(); + let first = librojo::snapshot_from_vfs(&ctx, &vfs, &project_path) + .expect("snapshot failed") + .expect("snapshot returned None"); + let baseline = snapshot_debug(&first); + + for i in 1..50 { + let vfs = memofs::Vfs::new_default(); + let snap = librojo::snapshot_from_vfs(&ctx, &vfs, &project_path) + .expect("snapshot failed") + .expect("snapshot returned None"); + let current = snapshot_debug(&snap); + assert_eq!( + baseline, current, + "Stress: parallel snapshot diverged on iteration {i}" + ); + } +} + +#[test] +fn parallel_snapshot_with_prefetch_cache() { + let _ = env_logger::try_init(); + + let dir = tempdir().expect("couldn't create temp dir"); + let root = dir.path(); + let src = root.join("src"); + fs::create_dir(&src).unwrap(); + + for i in 0..20 { + fs::write(src.join(format!("f{i}.luau")), format!("return {i}")).unwrap(); + } + for i in 0..5 { + let sub = src.join(format!("d{i}")); + fs::create_dir(&sub).unwrap(); + for j in 0..4 { + fs::write(sub.join(format!("g{j}.luau")), format!("return {j}")).unwrap(); + } + } + + fs::write( + root.join("default.project.json5"), + r#"{ "name": "PrefetchTest", "tree": { "$path": "src" } }"#, + ) + .unwrap(); + + let ctx = librojo::InstanceContext::default(); + let project_path = root.join("default.project.json5"); + + let vfs_no_cache = memofs::Vfs::new_default(); + let snap_no_cache = librojo::snapshot_from_vfs(&ctx, &vfs_no_cache, &project_path) + .expect("no-cache snapshot failed") + .expect("no-cache snapshot returned None"); + let baseline = snapshot_debug(&snap_no_cache); + + use std::collections::HashMap; + let mut files = HashMap::new(); + let mut canonical = HashMap::new(); + for entry in walkdir::WalkDir::new(root).follow_links(true) { + let entry = entry.unwrap(); + let path = entry.path().to_path_buf(); + if entry.file_type().is_file() { + files.insert(path.clone(), fs::read(&path).unwrap()); + } + if let Ok(c) = fs::canonicalize(&path) { + canonical.insert(path, c); + } + } + + let vfs_cached = memofs::Vfs::new_default(); + vfs_cached.set_prefetch_cache(memofs::PrefetchCache { + files, + canonical, + is_file: std::collections::HashMap::new(), + }); + + let snap_cached = librojo::snapshot_from_vfs(&ctx, &vfs_cached, &project_path) + .expect("cached snapshot failed") + .expect("cached snapshot returned None"); + let cached_str = snapshot_debug(&snap_cached); + + assert_eq!( + baseline, cached_str, + "Prefetch-cached snapshot differs from non-cached snapshot" + ); +} + +#[test] +fn parallel_snapshot_error_propagation() { + let _ = env_logger::try_init(); + + let dir = tempdir().expect("couldn't create temp dir"); + let root = dir.path(); + let src = root.join("src"); + fs::create_dir(&src).unwrap(); + + for i in 0..5 { + fs::write(src.join(format!("ok_{i}.luau")), format!("return {i}")).unwrap(); + } + fs::write(src.join("bad.model.json"), "THIS IS NOT VALID JSON {{{").unwrap(); + + fs::write( + root.join("default.project.json5"), + r#"{ "name": "ErrorTest", "tree": { "$path": "src" } }"#, + ) + .unwrap(); + + let ctx = librojo::InstanceContext::default(); + let project_path = root.join("default.project.json5"); + + let vfs = memofs::Vfs::new_default(); + let result = librojo::snapshot_from_vfs(&ctx, &vfs, &project_path); + assert!( + result.is_err(), + "Should propagate JSON parse error from parallel snapshot" + ); +}