use std::{ collections::{HashMap, HashSet}, path::Path, }; use anyhow::Context; use memofs::{DirEntry, Vfs}; use rbx_dom_weak::types::Ref; use crate::{ snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot, InstigatingSource}, syncback::{hash_instance, FsSnapshot, SyncbackReturn, SyncbackSnapshot}, }; use super::{meta_file::DirectoryMetadata, snapshot_from_vfs}; const EMPTY_DIR_KEEP_NAME: &str = ".gitkeep"; pub fn snapshot_dir( context: &InstanceContext, vfs: &Vfs, path: &Path, name: &str, ) -> anyhow::Result> { let mut snapshot = match snapshot_dir_no_meta(context, vfs, path, name)? { Some(snapshot) => snapshot, None => return Ok(None), }; DirectoryMetadata::read_and_apply_all(vfs, path, &mut snapshot)?; Ok(Some(snapshot)) } /// Snapshot a directory without applying meta files; useful for if the /// directory's ClassName will change before metadata should be applied. For /// example, this can happen if the directory contains an `init.client.luau` /// file. pub fn snapshot_dir_no_meta( context: &InstanceContext, vfs: &Vfs, path: &Path, name: &str, ) -> anyhow::Result> { let passes_filter_rules = |child: &DirEntry| { context .path_ignore_rules .iter() .all(|rule| rule.passes(child.path())) }; let mut any_child_glob_ignored = false; let snapshot_children = if std::env::var("ATLAS_SEQUENTIAL").is_ok() { let mut children = Vec::new(); for entry in vfs.read_dir(path)? { let entry = entry?; if !passes_filter_rules(&entry) { any_child_glob_ignored = true; continue; } if let Some(child_snapshot) = snapshot_from_vfs(context, vfs, entry.path())? { children.push(child_snapshot); } } children } else { use rayon::prelude::*; let entries: Vec<_> = vfs.read_dir(path)?.filter_map(|e| e.ok()).collect(); any_child_glob_ignored = entries.iter().any(|e| !passes_filter_rules(e)); let results: Vec>> = entries .par_iter() .map(|entry| { if !passes_filter_rules(entry) { return Ok(None); } snapshot_from_vfs(context, vfs, entry.path()) }) .collect(); let mut children = Vec::new(); for result in results { if let Some(snapshot) = result? { children.push(snapshot); } } children }; let normalized_path = path.to_path_buf(); let relevant_paths = vec![ normalized_path.clone(), // TODO: We shouldn't need to know about Lua existing in this // middleware. Should we figure out a way for that function to add // relevant paths to this middleware? // Modern extensions (preferred) normalized_path.join("init.luau"), normalized_path.join("init.server.luau"), normalized_path.join("init.client.luau"), normalized_path.join("init.local.luau"), normalized_path.join("init.legacy.luau"), normalized_path.join("init.csv"), // Legacy extensions (for backwards compatibility) normalized_path.join("init.lua"), normalized_path.join("init.server.lua"), normalized_path.join("init.client.lua"), ]; let snapshot = InstanceSnapshot::new() .name(name) .class_name("Folder") .children(snapshot_children) .metadata( InstanceMetadata::new() .instigating_source(path) .relevant_paths(relevant_paths) .context(context) .ignore_unknown_instances(any_child_glob_ignored) .glob_ignored_children(any_child_glob_ignored), ); Ok(Some(snapshot)) } pub fn syncback_dir<'sync>( snapshot: &SyncbackSnapshot<'sync>, ) -> anyhow::Result> { let new_inst = snapshot.new_inst(); let mut dir_syncback = syncback_dir_no_meta(snapshot)?; let mut meta = DirectoryMetadata::from_syncback_snapshot(snapshot, snapshot.path.clone())?; if let Some(meta) = &mut meta { if new_inst.class != "Folder" { meta.class_name = Some(new_inst.class); } if !meta.is_empty() { dir_syncback.fs_snapshot.add_file( snapshot.path.join("init.meta.json5"), crate::json::to_vec_pretty_sorted(&meta) .context("could not serialize new init.meta.json5")?, ); } } let metadata_empty = meta .as_ref() .map(DirectoryMetadata::is_empty) .unwrap_or_default(); if new_inst.children().is_empty() && metadata_empty { dir_syncback .fs_snapshot .add_file(snapshot.path.join(EMPTY_DIR_KEEP_NAME), Vec::new()) } Ok(dir_syncback) } pub fn syncback_dir_no_meta<'sync>( snapshot: &SyncbackSnapshot<'sync>, ) -> anyhow::Result> { use crate::syncback::matching::match_children; let new_inst = snapshot.new_inst(); let mut children = Vec::new(); let mut removed_children = Vec::new(); // taken_names tracks claimed full filesystem name components (slug + // extension for files, directory name for dirs), lowercased. let mut taken_names: HashSet = HashSet::new(); // Filter new children: exclude ignored classes and ignoreTrees. let eligible_new: Vec = new_inst .children() .iter() .copied() .filter(|r| { let child = snapshot.get_new_instance(*r).unwrap(); if snapshot.should_ignore_class(&child.class) { log::debug!( "Skipping instance {} because its class {} is ignored", child.name, child.class ); return false; } if snapshot.should_ignore_tree(*r) { return false; } true }) .collect(); if let Some(old_inst) = snapshot.old_inst() { // Build old_ref → InstanceWithMeta lookup for metadata access. let mut old_meta_map: HashMap = HashMap::new(); for &child_ref in old_inst.children() { let inst = snapshot.get_old_instance(child_ref).unwrap(); old_meta_map.insert(child_ref, inst); } // Filter old children: exclude those without disk presence, from // project files, or with ignored classes. let eligible_old: Vec = old_inst .children() .iter() .copied() .filter(|r| { let inst = old_meta_map.get(r).unwrap(); if inst.metadata().relevant_paths.is_empty() { log::debug!( "Excluding old instance {} from matching (no disk presence)", inst.name() ); return false; } if matches!( inst.metadata().instigating_source, Some(InstigatingSource::ProjectNode { .. }) ) { log::debug!( "Excluding old instance {} from matching (project file)", inst.name() ); return false; } if snapshot.should_ignore_class(inst.class_name().as_str()) { return false; } true }) .collect(); // Run the 3-pass matching algorithm to pair new ↔ old children. let matching_session = crate::syncback::matching::MatchingSession::new(); let match_result = match_children( &eligible_new, &eligible_old, snapshot.new_tree(), snapshot.old_tree(), None, // TODO: pass precomputed hashes for better similarity None, &matching_session, ); // Pre-seed taken_names from ALL matched old children's filesystem // names so that unmatched new children correctly dedup. if snapshot.data.is_incremental() { for &(_, old_ref) in &match_result.matched { if let Some(inst) = old_meta_map.get(&old_ref) { if let Some(path) = inst.metadata().relevant_paths.first() { if let Some(filename) = path.file_name().and_then(|f| f.to_str()) { taken_names.insert(filename.to_lowercase()); } } } } // Also seed from unmatched old (they still occupy filesystem names). for &old_ref in &match_result.unmatched_old { if let Some(inst) = old_meta_map.get(&old_ref) { if let Some(path) = inst.metadata().relevant_paths.first() { if let Some(filename) = path.file_name().and_then(|f| f.to_str()) { taken_names.insert(filename.to_lowercase()); } } } } } // Sort matched pairs by new child name for deterministic ordering. let mut sorted_matched = match_result.matched; sorted_matched.sort_by(|(a, _), (b, _)| { let a_name = &snapshot.get_new_instance(*a).unwrap().name; let b_name = &snapshot.get_new_instance(*b).unwrap().name; a_name.cmp(b_name) }); // Process matched pairs: preserve existing filesystem assignment. for (new_ref, old_ref) in &sorted_matched { let (child_snap, _needs_meta, dedup_key) = snapshot.with_joined_path(*new_ref, Some(*old_ref), &taken_names)?; taken_names.insert(dedup_key.to_lowercase()); children.push(child_snap); } // Sort unmatched new by name for deterministic dedup ordering. let mut sorted_unmatched_new = match_result.unmatched_new; sorted_unmatched_new.sort_by(|a, b| { let a_name = &snapshot.get_new_instance(*a).unwrap().name; let b_name = &snapshot.get_new_instance(*b).unwrap().name; a_name.cmp(b_name) }); // Process unmatched new children: generate new filenames. for new_ref in &sorted_unmatched_new { let (child_snap, _needs_meta, dedup_key) = snapshot.with_joined_path(*new_ref, None, &taken_names)?; taken_names.insert(dedup_key.to_lowercase()); children.push(child_snap); } // Unmatched old children are removed (instance was deleted). // Filter out ignored classes from removal. for old_ref in match_result.unmatched_old { if let Some(inst) = old_meta_map.get(&old_ref) { if !snapshot.should_ignore_class(inst.class_name().as_str()) { removed_children.push(*inst); } else { log::debug!( "Not removing instance {} because its class {} is ignored", inst.name(), inst.class_name() ); } } } } else { // No old instance (clean mode). All children are new. // Sort alphabetically for deterministic dedup ordering. let mut sorted_new = eligible_new; sorted_new.sort_by(|a, b| { let a_name = &snapshot.get_new_instance(*a).unwrap().name; let b_name = &snapshot.get_new_instance(*b).unwrap().name; a_name.cmp(b_name) }); for new_child_ref in &sorted_new { let (child_snap, _needs_meta, dedup_key) = snapshot.with_joined_path(*new_child_ref, None, &taken_names)?; taken_names.insert(dedup_key.to_lowercase()); children.push(child_snap); } } let mut fs_snapshot = FsSnapshot::new(); if let Some(old_ref) = snapshot.old { let new_hash = hash_instance(snapshot.project(), snapshot.new_tree(), snapshot.new) .expect("new Instance should be hashable"); let old_hash = hash_instance(snapshot.project(), snapshot.old_tree(), old_ref) .expect("old Instance should be hashable"); if old_hash != new_hash { fs_snapshot.add_dir(&snapshot.path); } else { log::debug!( "Skipping reserializing directory {} because old and new tree hash the same", new_inst.name ); } } else { fs_snapshot.add_dir(&snapshot.path); } Ok(SyncbackReturn { fs_snapshot, children, removed_children, }) } #[cfg(test)] mod test { use super::*; use memofs::{InMemoryFs, VfsSnapshot}; #[test] fn parallel_wide_directory_ordering() { let children: Vec<_> = (0..50) .map(|i| { ( format!("child_{i:03}.luau"), VfsSnapshot::file(format!("-- {i}")), ) }) .collect(); let mut imfs = InMemoryFs::new(); imfs.load_snapshot("/wide", VfsSnapshot::dir(children)) .unwrap(); let vfs = Vfs::new(imfs); let snap = snapshot_dir( &InstanceContext::default(), &vfs, Path::new("/wide"), "wide", ) .unwrap() .unwrap(); let names: Vec<&str> = snap.children.iter().map(|c| c.name.as_ref()).collect(); let mut sorted = names.clone(); sorted.sort(); assert_eq!( names, sorted, "Children should be in sorted (read_dir) order" ); assert_eq!(snap.children.len(), 50); } #[test] fn parallel_wide_directory_deterministic_across_runs() { fn make_children() -> Vec<(String, VfsSnapshot)> { (0..30) .map(|i| { ( format!("mod_{i:03}.luau"), VfsSnapshot::file(format!("-- {i}")), ) }) .collect() } fn names_from_fresh() -> Vec { let mut imfs = InMemoryFs::new(); imfs.load_snapshot("/root", VfsSnapshot::dir(make_children())) .unwrap(); let vfs = Vfs::new(imfs); let snap = snapshot_dir( &InstanceContext::default(), &vfs, Path::new("/root"), "root", ) .unwrap() .unwrap(); snap.children.iter().map(|c| c.name.to_string()).collect() } let baseline = names_from_fresh(); for i in 0..20 { assert_eq!( baseline, names_from_fresh(), "Child order diverged on iteration {i}" ); } } #[test] fn parallel_deep_wide_tree() { fn make_tree(depth: usize, width: usize) -> VfsSnapshot { if depth == 0 { return VfsSnapshot::file("-- leaf"); } let children: Vec<_> = (0..width) .map(|i| { if i % 3 == 0 { (format!("dir_{i}"), make_tree(depth - 1, width)) } else { ( format!("file_{i}.luau"), VfsSnapshot::file(format!("-- d{depth} f{i}")), ) } }) .collect(); VfsSnapshot::dir(children) } let mut imfs = InMemoryFs::new(); imfs.load_snapshot("/deep", make_tree(4, 8)).unwrap(); let vfs = Vfs::new(imfs); let snap = snapshot_dir( &InstanceContext::default(), &vfs, Path::new("/deep"), "deep", ) .unwrap() .unwrap(); fn count_nodes(snap: &InstanceSnapshot) -> usize { 1 + snap.children.iter().map(count_nodes).sum::() } let total = count_nodes(&snap); assert!(total > 100, "Should have many nodes, got {total}"); } #[test] fn parallel_mixed_middleware_types() { let mut imfs = InMemoryFs::new(); imfs.load_snapshot( "/mixed", VfsSnapshot::dir([ ("script.server.luau", VfsSnapshot::file("-- server")), ("module.luau", VfsSnapshot::file("return {}")), ("data.json", VfsSnapshot::file(r#"{"key": "value"}"#)), ("notes.txt", VfsSnapshot::file("hello")), ( "sub", VfsSnapshot::dir([ ("inner.luau", VfsSnapshot::file("return 1")), ( "deep", VfsSnapshot::dir([("leaf.luau", VfsSnapshot::file("return 2"))]), ), ]), ), ("another.luau", VfsSnapshot::file("return 3")), ( "model.model.json", VfsSnapshot::file(r#"{"className": "Part", "properties": {}}"#), ), ]), ) .unwrap(); let vfs = Vfs::new(imfs); let snap = snapshot_dir( &InstanceContext::default(), &vfs, Path::new("/mixed"), "mixed", ) .unwrap() .unwrap(); let names: Vec<&str> = snap.children.iter().map(|c| c.name.as_ref()).collect(); assert!(names.contains(&"script"), "Missing server script"); assert!(names.contains(&"module"), "Missing module script"); assert!(names.contains(&"data"), "Missing json data"); assert!(names.contains(&"notes"), "Missing txt"); assert!(names.contains(&"sub"), "Missing subdirectory"); assert!(names.contains(&"another"), "Missing another script"); assert!(names.contains(&"model"), "Missing model"); let sub = snap.children.iter().find(|c| c.name == "sub").unwrap(); assert_eq!(sub.children.len(), 2, "sub should have inner + deep"); } #[test] fn empty_folder() { let mut imfs = InMemoryFs::new(); imfs.load_snapshot("/foo", VfsSnapshot::empty_dir()) .unwrap(); let vfs = Vfs::new(imfs); let instance_snapshot = snapshot_dir(&InstanceContext::default(), &vfs, Path::new("/foo"), "foo") .unwrap() .unwrap(); insta::assert_yaml_snapshot!(instance_snapshot); } #[test] fn folder_in_folder() { let mut imfs = InMemoryFs::new(); imfs.load_snapshot( "/foo", VfsSnapshot::dir([("Child", VfsSnapshot::empty_dir())]), ) .unwrap(); let vfs = Vfs::new(imfs); let instance_snapshot = snapshot_dir(&InstanceContext::default(), &vfs, Path::new("/foo"), "foo") .unwrap() .unwrap(); insta::assert_yaml_snapshot!(instance_snapshot); } #[test] fn glob_ignored_children_set_ignore_unknown_instances() { use crate::glob::Glob; use crate::snapshot::PathIgnoreRule; use std::path::PathBuf; let mut imfs = InMemoryFs::new(); imfs.load_snapshot( "/project/src", VfsSnapshot::dir([ ("kept.luau", VfsSnapshot::file("return 1")), ("ignored.luau", VfsSnapshot::file("return 2")), ]), ) .unwrap(); let vfs = Vfs::new(imfs); let mut context = InstanceContext::default(); context.add_path_ignore_rules(vec![PathIgnoreRule { base_path: PathBuf::from("/project"), glob: Glob::new("src/ignored.luau").unwrap(), }]); let snap = snapshot_dir(&context, &vfs, Path::new("/project/src"), "src") .unwrap() .unwrap(); assert!( snap.metadata.ignore_unknown_instances, "Directory with glob-ignored children should have ignore_unknown_instances = true" ); assert!( snap.metadata.glob_ignored_children, "Directory with glob-ignored children should have glob_ignored_children = true" ); assert_eq!( snap.children.len(), 1, "Only non-ignored child should be snapshotted" ); assert_eq!(snap.children[0].name, "kept"); } #[test] fn no_glob_ignored_children_leaves_ignore_unknown_instances_false() { let mut imfs = InMemoryFs::new(); imfs.load_snapshot( "/project/src", VfsSnapshot::dir([ ("a.luau", VfsSnapshot::file("return 1")), ("b.luau", VfsSnapshot::file("return 2")), ]), ) .unwrap(); let vfs = Vfs::new(imfs); let snap = snapshot_dir( &InstanceContext::default(), &vfs, Path::new("/project/src"), "src", ) .unwrap() .unwrap(); assert!( !snap.metadata.ignore_unknown_instances, "Directory without glob-ignored children should have ignore_unknown_instances = false" ); assert!( !snap.metadata.glob_ignored_children, "Directory without glob-ignored children should have glob_ignored_children = false" ); assert_eq!(snap.children.len(), 2); } }