Enhance PrefetchCache functionality and improve project file prefetching

- Updated the `PrefetchCache` structure to include metadata about whether paths are files or directories, optimizing the cache's utility during VFS operations.
- Refactored the `prefetch_project_files` function to accept a `Project` object, allowing for more flexible path collection and improved logging of the prefetching process.
- Added tests to validate the behavior of the prefetch cache, ensuring correct handling of file reads, canonicalization, and cache depletion scenarios.
- Enhanced existing tests to cover new functionality, including concurrent reads and stress tests for snapshot determinism.

These changes aim to improve the performance and reliability of the VFS by refining the prefetching mechanism and ensuring accurate metadata handling.

tests/tests/build.rs

@@ -114,6 +114,20 @@ fn run_build_test(test_name: &str) {
     });
 }
 
+fn snapshot_debug(snap: &librojo::InstanceSnapshot) -> String {
+    fn recurse(snap: &librojo::InstanceSnapshot, depth: usize, out: &mut String) {
+        use std::fmt::Write;
+        let indent = "  ".repeat(depth);
+        writeln!(out, "{}{} [{}]", indent, snap.name, snap.class_name).unwrap();
+        for child in &snap.children {
+            recurse(child, depth + 1, out);
+        }
+    }
+    let mut s = String::new();
+    recurse(snap, 0, &mut s);
+    s
+}
+
 #[test]
 fn parallel_snapshot_determinism() {
     use crate::rojo_test::io_util::SERVE_TESTS_PATH;
@@ -124,20 +138,6 @@ fn parallel_snapshot_determinism() {
     let vfs = memofs::Vfs::new_default();
     let ctx = librojo::InstanceContext::default();
 
-    fn snapshot_debug(snap: &librojo::InstanceSnapshot) -> String {
-        fn recurse(snap: &librojo::InstanceSnapshot, depth: usize, out: &mut String) {
-            use std::fmt::Write;
-            let indent = "  ".repeat(depth);
-            writeln!(out, "{}{} [{}]", indent, snap.name, snap.class_name).unwrap();
-            for child in &snap.children {
-                recurse(child, depth + 1, out);
-            }
-        }
-        let mut s = String::new();
-        recurse(snap, 0, &mut s);
-        s
-    }
-
     let first = librojo::snapshot_from_vfs(&ctx, &vfs, &fixture_path)
         .expect("snapshot failed")
         .expect("snapshot returned None");
@@ -155,3 +155,149 @@ fn parallel_snapshot_determinism() {
         );
     }
 }
+
+#[test]
+fn parallel_snapshot_determinism_stress() {
+    let _ = env_logger::try_init();
+
+    let dir = tempdir().expect("couldn't create temp dir");
+    let root = dir.path();
+    let src = root.join("src");
+    fs::create_dir(&src).unwrap();
+
+    for i in 0..40 {
+        fs::write(src.join(format!("mod_{i:03}.luau")), format!("return {i}")).unwrap();
+    }
+    for i in 0..10 {
+        let sub = src.join(format!("pkg_{i:02}"));
+        fs::create_dir(&sub).unwrap();
+        for j in 0..5 {
+            fs::write(sub.join(format!("sub_{j}.luau")), format!("return {j}")).unwrap();
+        }
+    }
+
+    fs::write(
+        root.join("default.project.json5"),
+        r#"{ "name": "StressTest", "tree": { "$path": "src" } }"#,
+    )
+    .unwrap();
+
+    let ctx = librojo::InstanceContext::default();
+    let project_path = root.join("default.project.json5");
+
+    let vfs = memofs::Vfs::new_default();
+    let first = librojo::snapshot_from_vfs(&ctx, &vfs, &project_path)
+        .expect("snapshot failed")
+        .expect("snapshot returned None");
+    let baseline = snapshot_debug(&first);
+
+    for i in 1..50 {
+        let vfs = memofs::Vfs::new_default();
+        let snap = librojo::snapshot_from_vfs(&ctx, &vfs, &project_path)
+            .expect("snapshot failed")
+            .expect("snapshot returned None");
+        let current = snapshot_debug(&snap);
+        assert_eq!(
+            baseline, current,
+            "Stress: parallel snapshot diverged on iteration {i}"
+        );
+    }
+}
+
+#[test]
+fn parallel_snapshot_with_prefetch_cache() {
+    let _ = env_logger::try_init();
+
+    let dir = tempdir().expect("couldn't create temp dir");
+    let root = dir.path();
+    let src = root.join("src");
+    fs::create_dir(&src).unwrap();
+
+    for i in 0..20 {
+        fs::write(src.join(format!("f{i}.luau")), format!("return {i}")).unwrap();
+    }
+    for i in 0..5 {
+        let sub = src.join(format!("d{i}"));
+        fs::create_dir(&sub).unwrap();
+        for j in 0..4 {
+            fs::write(sub.join(format!("g{j}.luau")), format!("return {j}")).unwrap();
+        }
+    }
+
+    fs::write(
+        root.join("default.project.json5"),
+        r#"{ "name": "PrefetchTest", "tree": { "$path": "src" } }"#,
+    )
+    .unwrap();
+
+    let ctx = librojo::InstanceContext::default();
+    let project_path = root.join("default.project.json5");
+
+    let vfs_no_cache = memofs::Vfs::new_default();
+    let snap_no_cache = librojo::snapshot_from_vfs(&ctx, &vfs_no_cache, &project_path)
+        .expect("no-cache snapshot failed")
+        .expect("no-cache snapshot returned None");
+    let baseline = snapshot_debug(&snap_no_cache);
+
+    use std::collections::HashMap;
+    let mut files = HashMap::new();
+    let mut canonical = HashMap::new();
+    for entry in walkdir::WalkDir::new(root).follow_links(true) {
+        let entry = entry.unwrap();
+        let path = entry.path().to_path_buf();
+        if entry.file_type().is_file() {
+            files.insert(path.clone(), fs::read(&path).unwrap());
+        }
+        if let Ok(c) = fs::canonicalize(&path) {
+            canonical.insert(path, c);
+        }
+    }
+
+    let vfs_cached = memofs::Vfs::new_default();
+    vfs_cached.set_prefetch_cache(memofs::PrefetchCache {
+        files,
+        canonical,
+        is_file: std::collections::HashMap::new(),
+    });
+
+    let snap_cached = librojo::snapshot_from_vfs(&ctx, &vfs_cached, &project_path)
+        .expect("cached snapshot failed")
+        .expect("cached snapshot returned None");
+    let cached_str = snapshot_debug(&snap_cached);
+
+    assert_eq!(
+        baseline, cached_str,
+        "Prefetch-cached snapshot differs from non-cached snapshot"
+    );
+}
+
+#[test]
+fn parallel_snapshot_error_propagation() {
+    let _ = env_logger::try_init();
+
+    let dir = tempdir().expect("couldn't create temp dir");
+    let root = dir.path();
+    let src = root.join("src");
+    fs::create_dir(&src).unwrap();
+
+    for i in 0..5 {
+        fs::write(src.join(format!("ok_{i}.luau")), format!("return {i}")).unwrap();
+    }
+    fs::write(src.join("bad.model.json"), "THIS IS NOT VALID JSON {{{").unwrap();
+
+    fs::write(
+        root.join("default.project.json5"),
+        r#"{ "name": "ErrorTest", "tree": { "$path": "src" } }"#,
+    )
+    .unwrap();
+
+    let ctx = librojo::InstanceContext::default();
+    let project_path = root.join("default.project.json5");
+
+    let vfs = memofs::Vfs::new_default();
+    let result = librojo::snapshot_from_vfs(&ctx, &vfs, &project_path);
+    assert!(
+        result.is_err(),
+        "Should propagate JSON parse error from parallel snapshot"
+    );
+}