rojo/src/snapshot_middleware/dir.rs
Joe 12bccf91c5 Enhance instance metadata and reconciliation logic for glob-ignored children
- Added `globIgnoredChildren` to `ApiInstanceMetadata` to manage child deletion behavior based on glob patterns.
- Updated `shouldDeleteChild` function to suppress deletions when `globIgnoredChildren` is true, differentiating from scripts-only mode.
- Enhanced tests to verify correct behavior of instance deletion based on the new metadata, ensuring robust handling of unknown scripts.
- Refactored snapshot handling to incorporate `globIgnoredChildren` in metadata, improving clarity and maintainability.
2026-03-07 11:44:07 -06:00

650 lines
22 KiB
Rust

use std::{
collections::{HashMap, HashSet},
path::Path,
};
use anyhow::Context;
use memofs::{DirEntry, Vfs};
use rbx_dom_weak::types::Ref;
use crate::{
snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot, InstigatingSource},
syncback::{hash_instance, FsSnapshot, SyncbackReturn, SyncbackSnapshot},
};
use super::{meta_file::DirectoryMetadata, snapshot_from_vfs};
const EMPTY_DIR_KEEP_NAME: &str = ".gitkeep";
pub fn snapshot_dir(
context: &InstanceContext,
vfs: &Vfs,
path: &Path,
name: &str,
) -> anyhow::Result<Option<InstanceSnapshot>> {
let mut snapshot = match snapshot_dir_no_meta(context, vfs, path, name)? {
Some(snapshot) => snapshot,
None => return Ok(None),
};
DirectoryMetadata::read_and_apply_all(vfs, path, &mut snapshot)?;
Ok(Some(snapshot))
}
/// Snapshot a directory without applying meta files; useful for if the
/// directory's ClassName will change before metadata should be applied. For
/// example, this can happen if the directory contains an `init.client.luau`
/// file.
pub fn snapshot_dir_no_meta(
context: &InstanceContext,
vfs: &Vfs,
path: &Path,
name: &str,
) -> anyhow::Result<Option<InstanceSnapshot>> {
let passes_filter_rules = |child: &DirEntry| {
context
.path_ignore_rules
.iter()
.all(|rule| rule.passes(child.path()))
};
let mut any_child_glob_ignored = false;
let snapshot_children = if std::env::var("ATLAS_SEQUENTIAL").is_ok() {
let mut children = Vec::new();
for entry in vfs.read_dir(path)? {
let entry = entry?;
if !passes_filter_rules(&entry) {
any_child_glob_ignored = true;
continue;
}
if let Some(child_snapshot) = snapshot_from_vfs(context, vfs, entry.path())? {
children.push(child_snapshot);
}
}
children
} else {
use rayon::prelude::*;
let entries: Vec<_> = vfs.read_dir(path)?.filter_map(|e| e.ok()).collect();
any_child_glob_ignored = entries.iter().any(|e| !passes_filter_rules(e));
let results: Vec<anyhow::Result<Option<InstanceSnapshot>>> = entries
.par_iter()
.map(|entry| {
if !passes_filter_rules(entry) {
return Ok(None);
}
snapshot_from_vfs(context, vfs, entry.path())
})
.collect();
let mut children = Vec::new();
for result in results {
if let Some(snapshot) = result? {
children.push(snapshot);
}
}
children
};
let normalized_path = path.to_path_buf();
let relevant_paths = vec![
normalized_path.clone(),
// TODO: We shouldn't need to know about Lua existing in this
// middleware. Should we figure out a way for that function to add
// relevant paths to this middleware?
// Modern extensions (preferred)
normalized_path.join("init.luau"),
normalized_path.join("init.server.luau"),
normalized_path.join("init.client.luau"),
normalized_path.join("init.local.luau"),
normalized_path.join("init.legacy.luau"),
normalized_path.join("init.csv"),
// Legacy extensions (for backwards compatibility)
normalized_path.join("init.lua"),
normalized_path.join("init.server.lua"),
normalized_path.join("init.client.lua"),
];
let snapshot = InstanceSnapshot::new()
.name(name)
.class_name("Folder")
.children(snapshot_children)
.metadata(
InstanceMetadata::new()
.instigating_source(path)
.relevant_paths(relevant_paths)
.context(context)
.ignore_unknown_instances(any_child_glob_ignored)
.glob_ignored_children(any_child_glob_ignored),
);
Ok(Some(snapshot))
}
pub fn syncback_dir<'sync>(
snapshot: &SyncbackSnapshot<'sync>,
) -> anyhow::Result<SyncbackReturn<'sync>> {
let new_inst = snapshot.new_inst();
let mut dir_syncback = syncback_dir_no_meta(snapshot)?;
let mut meta = DirectoryMetadata::from_syncback_snapshot(snapshot, snapshot.path.clone())?;
if let Some(meta) = &mut meta {
if new_inst.class != "Folder" {
meta.class_name = Some(new_inst.class);
}
if !meta.is_empty() {
dir_syncback.fs_snapshot.add_file(
snapshot.path.join("init.meta.json5"),
crate::json::to_vec_pretty_sorted(&meta)
.context("could not serialize new init.meta.json5")?,
);
}
}
let metadata_empty = meta
.as_ref()
.map(DirectoryMetadata::is_empty)
.unwrap_or_default();
if new_inst.children().is_empty() && metadata_empty {
dir_syncback
.fs_snapshot
.add_file(snapshot.path.join(EMPTY_DIR_KEEP_NAME), Vec::new())
}
Ok(dir_syncback)
}
pub fn syncback_dir_no_meta<'sync>(
snapshot: &SyncbackSnapshot<'sync>,
) -> anyhow::Result<SyncbackReturn<'sync>> {
use crate::syncback::matching::match_children;
let new_inst = snapshot.new_inst();
let mut children = Vec::new();
let mut removed_children = Vec::new();
// taken_names tracks claimed full filesystem name components (slug +
// extension for files, directory name for dirs), lowercased.
let mut taken_names: HashSet<String> = HashSet::new();
// Filter new children: exclude ignored classes and ignoreTrees.
let eligible_new: Vec<Ref> = new_inst
.children()
.iter()
.copied()
.filter(|r| {
let child = snapshot.get_new_instance(*r).unwrap();
if snapshot.should_ignore_class(&child.class) {
log::debug!(
"Skipping instance {} because its class {} is ignored",
child.name,
child.class
);
return false;
}
if snapshot.should_ignore_tree(*r) {
return false;
}
true
})
.collect();
if let Some(old_inst) = snapshot.old_inst() {
// Build old_ref → InstanceWithMeta lookup for metadata access.
let mut old_meta_map: HashMap<Ref, _> = HashMap::new();
for &child_ref in old_inst.children() {
let inst = snapshot.get_old_instance(child_ref).unwrap();
old_meta_map.insert(child_ref, inst);
}
// Filter old children: exclude those without disk presence, from
// project files, or with ignored classes.
let eligible_old: Vec<Ref> = old_inst
.children()
.iter()
.copied()
.filter(|r| {
let inst = old_meta_map.get(r).unwrap();
if inst.metadata().relevant_paths.is_empty() {
log::debug!(
"Excluding old instance {} from matching (no disk presence)",
inst.name()
);
return false;
}
if matches!(
inst.metadata().instigating_source,
Some(InstigatingSource::ProjectNode { .. })
) {
log::debug!(
"Excluding old instance {} from matching (project file)",
inst.name()
);
return false;
}
if snapshot.should_ignore_class(inst.class_name().as_str()) {
return false;
}
true
})
.collect();
// Run the 3-pass matching algorithm to pair new ↔ old children.
let matching_session = crate::syncback::matching::MatchingSession::new();
let match_result = match_children(
&eligible_new,
&eligible_old,
snapshot.new_tree(),
snapshot.old_tree(),
None, // TODO: pass precomputed hashes for better similarity
None,
&matching_session,
);
// Pre-seed taken_names from ALL matched old children's filesystem
// names so that unmatched new children correctly dedup.
if snapshot.data.is_incremental() {
for &(_, old_ref) in &match_result.matched {
if let Some(inst) = old_meta_map.get(&old_ref) {
if let Some(path) = inst.metadata().relevant_paths.first() {
if let Some(filename) = path.file_name().and_then(|f| f.to_str()) {
taken_names.insert(filename.to_lowercase());
}
}
}
}
// Also seed from unmatched old (they still occupy filesystem names).
for &old_ref in &match_result.unmatched_old {
if let Some(inst) = old_meta_map.get(&old_ref) {
if let Some(path) = inst.metadata().relevant_paths.first() {
if let Some(filename) = path.file_name().and_then(|f| f.to_str()) {
taken_names.insert(filename.to_lowercase());
}
}
}
}
}
// Sort matched pairs by new child name for deterministic ordering.
let mut sorted_matched = match_result.matched;
sorted_matched.sort_by(|(a, _), (b, _)| {
let a_name = &snapshot.get_new_instance(*a).unwrap().name;
let b_name = &snapshot.get_new_instance(*b).unwrap().name;
a_name.cmp(b_name)
});
// Process matched pairs: preserve existing filesystem assignment.
for (new_ref, old_ref) in &sorted_matched {
let (child_snap, _needs_meta, dedup_key) =
snapshot.with_joined_path(*new_ref, Some(*old_ref), &taken_names)?;
taken_names.insert(dedup_key.to_lowercase());
children.push(child_snap);
}
// Sort unmatched new by name for deterministic dedup ordering.
let mut sorted_unmatched_new = match_result.unmatched_new;
sorted_unmatched_new.sort_by(|a, b| {
let a_name = &snapshot.get_new_instance(*a).unwrap().name;
let b_name = &snapshot.get_new_instance(*b).unwrap().name;
a_name.cmp(b_name)
});
// Process unmatched new children: generate new filenames.
for new_ref in &sorted_unmatched_new {
let (child_snap, _needs_meta, dedup_key) =
snapshot.with_joined_path(*new_ref, None, &taken_names)?;
taken_names.insert(dedup_key.to_lowercase());
children.push(child_snap);
}
// Unmatched old children are removed (instance was deleted).
// Filter out ignored classes from removal.
for old_ref in match_result.unmatched_old {
if let Some(inst) = old_meta_map.get(&old_ref) {
if !snapshot.should_ignore_class(inst.class_name().as_str()) {
removed_children.push(*inst);
} else {
log::debug!(
"Not removing instance {} because its class {} is ignored",
inst.name(),
inst.class_name()
);
}
}
}
} else {
// No old instance (clean mode). All children are new.
// Sort alphabetically for deterministic dedup ordering.
let mut sorted_new = eligible_new;
sorted_new.sort_by(|a, b| {
let a_name = &snapshot.get_new_instance(*a).unwrap().name;
let b_name = &snapshot.get_new_instance(*b).unwrap().name;
a_name.cmp(b_name)
});
for new_child_ref in &sorted_new {
let (child_snap, _needs_meta, dedup_key) =
snapshot.with_joined_path(*new_child_ref, None, &taken_names)?;
taken_names.insert(dedup_key.to_lowercase());
children.push(child_snap);
}
}
let mut fs_snapshot = FsSnapshot::new();
if let Some(old_ref) = snapshot.old {
let new_hash = hash_instance(snapshot.project(), snapshot.new_tree(), snapshot.new)
.expect("new Instance should be hashable");
let old_hash = hash_instance(snapshot.project(), snapshot.old_tree(), old_ref)
.expect("old Instance should be hashable");
if old_hash != new_hash {
fs_snapshot.add_dir(&snapshot.path);
} else {
log::debug!(
"Skipping reserializing directory {} because old and new tree hash the same",
new_inst.name
);
}
} else {
fs_snapshot.add_dir(&snapshot.path);
}
Ok(SyncbackReturn {
fs_snapshot,
children,
removed_children,
})
}
#[cfg(test)]
mod test {
use super::*;
use memofs::{InMemoryFs, VfsSnapshot};
#[test]
fn parallel_wide_directory_ordering() {
let children: Vec<_> = (0..50)
.map(|i| {
(
format!("child_{i:03}.luau"),
VfsSnapshot::file(format!("-- {i}")),
)
})
.collect();
let mut imfs = InMemoryFs::new();
imfs.load_snapshot("/wide", VfsSnapshot::dir(children))
.unwrap();
let vfs = Vfs::new(imfs);
let snap = snapshot_dir(
&InstanceContext::default(),
&vfs,
Path::new("/wide"),
"wide",
)
.unwrap()
.unwrap();
let names: Vec<&str> = snap.children.iter().map(|c| c.name.as_ref()).collect();
let mut sorted = names.clone();
sorted.sort();
assert_eq!(
names, sorted,
"Children should be in sorted (read_dir) order"
);
assert_eq!(snap.children.len(), 50);
}
#[test]
fn parallel_wide_directory_deterministic_across_runs() {
fn make_children() -> Vec<(String, VfsSnapshot)> {
(0..30)
.map(|i| {
(
format!("mod_{i:03}.luau"),
VfsSnapshot::file(format!("-- {i}")),
)
})
.collect()
}
fn names_from_fresh() -> Vec<String> {
let mut imfs = InMemoryFs::new();
imfs.load_snapshot("/root", VfsSnapshot::dir(make_children()))
.unwrap();
let vfs = Vfs::new(imfs);
let snap = snapshot_dir(
&InstanceContext::default(),
&vfs,
Path::new("/root"),
"root",
)
.unwrap()
.unwrap();
snap.children.iter().map(|c| c.name.to_string()).collect()
}
let baseline = names_from_fresh();
for i in 0..20 {
assert_eq!(
baseline,
names_from_fresh(),
"Child order diverged on iteration {i}"
);
}
}
#[test]
fn parallel_deep_wide_tree() {
fn make_tree(depth: usize, width: usize) -> VfsSnapshot {
if depth == 0 {
return VfsSnapshot::file("-- leaf");
}
let children: Vec<_> = (0..width)
.map(|i| {
if i % 3 == 0 {
(format!("dir_{i}"), make_tree(depth - 1, width))
} else {
(
format!("file_{i}.luau"),
VfsSnapshot::file(format!("-- d{depth} f{i}")),
)
}
})
.collect();
VfsSnapshot::dir(children)
}
let mut imfs = InMemoryFs::new();
imfs.load_snapshot("/deep", make_tree(4, 8)).unwrap();
let vfs = Vfs::new(imfs);
let snap = snapshot_dir(
&InstanceContext::default(),
&vfs,
Path::new("/deep"),
"deep",
)
.unwrap()
.unwrap();
fn count_nodes(snap: &InstanceSnapshot) -> usize {
1 + snap.children.iter().map(count_nodes).sum::<usize>()
}
let total = count_nodes(&snap);
assert!(total > 100, "Should have many nodes, got {total}");
}
#[test]
fn parallel_mixed_middleware_types() {
let mut imfs = InMemoryFs::new();
imfs.load_snapshot(
"/mixed",
VfsSnapshot::dir([
("script.server.luau", VfsSnapshot::file("-- server")),
("module.luau", VfsSnapshot::file("return {}")),
("data.json", VfsSnapshot::file(r#"{"key": "value"}"#)),
("notes.txt", VfsSnapshot::file("hello")),
(
"sub",
VfsSnapshot::dir([
("inner.luau", VfsSnapshot::file("return 1")),
(
"deep",
VfsSnapshot::dir([("leaf.luau", VfsSnapshot::file("return 2"))]),
),
]),
),
("another.luau", VfsSnapshot::file("return 3")),
(
"model.model.json",
VfsSnapshot::file(r#"{"className": "Part", "properties": {}}"#),
),
]),
)
.unwrap();
let vfs = Vfs::new(imfs);
let snap = snapshot_dir(
&InstanceContext::default(),
&vfs,
Path::new("/mixed"),
"mixed",
)
.unwrap()
.unwrap();
let names: Vec<&str> = snap.children.iter().map(|c| c.name.as_ref()).collect();
assert!(names.contains(&"script"), "Missing server script");
assert!(names.contains(&"module"), "Missing module script");
assert!(names.contains(&"data"), "Missing json data");
assert!(names.contains(&"notes"), "Missing txt");
assert!(names.contains(&"sub"), "Missing subdirectory");
assert!(names.contains(&"another"), "Missing another script");
assert!(names.contains(&"model"), "Missing model");
let sub = snap.children.iter().find(|c| c.name == "sub").unwrap();
assert_eq!(sub.children.len(), 2, "sub should have inner + deep");
}
#[test]
fn empty_folder() {
let mut imfs = InMemoryFs::new();
imfs.load_snapshot("/foo", VfsSnapshot::empty_dir())
.unwrap();
let vfs = Vfs::new(imfs);
let instance_snapshot =
snapshot_dir(&InstanceContext::default(), &vfs, Path::new("/foo"), "foo")
.unwrap()
.unwrap();
insta::assert_yaml_snapshot!(instance_snapshot);
}
#[test]
fn folder_in_folder() {
let mut imfs = InMemoryFs::new();
imfs.load_snapshot(
"/foo",
VfsSnapshot::dir([("Child", VfsSnapshot::empty_dir())]),
)
.unwrap();
let vfs = Vfs::new(imfs);
let instance_snapshot =
snapshot_dir(&InstanceContext::default(), &vfs, Path::new("/foo"), "foo")
.unwrap()
.unwrap();
insta::assert_yaml_snapshot!(instance_snapshot);
}
#[test]
fn glob_ignored_children_set_ignore_unknown_instances() {
use crate::glob::Glob;
use crate::snapshot::PathIgnoreRule;
use std::path::PathBuf;
let mut imfs = InMemoryFs::new();
imfs.load_snapshot(
"/project/src",
VfsSnapshot::dir([
("kept.luau", VfsSnapshot::file("return 1")),
("ignored.luau", VfsSnapshot::file("return 2")),
]),
)
.unwrap();
let vfs = Vfs::new(imfs);
let mut context = InstanceContext::default();
context.add_path_ignore_rules(vec![PathIgnoreRule {
base_path: PathBuf::from("/project"),
glob: Glob::new("src/ignored.luau").unwrap(),
}]);
let snap = snapshot_dir(&context, &vfs, Path::new("/project/src"), "src")
.unwrap()
.unwrap();
assert!(
snap.metadata.ignore_unknown_instances,
"Directory with glob-ignored children should have ignore_unknown_instances = true"
);
assert!(
snap.metadata.glob_ignored_children,
"Directory with glob-ignored children should have glob_ignored_children = true"
);
assert_eq!(
snap.children.len(),
1,
"Only non-ignored child should be snapshotted"
);
assert_eq!(snap.children[0].name, "kept");
}
#[test]
fn no_glob_ignored_children_leaves_ignore_unknown_instances_false() {
let mut imfs = InMemoryFs::new();
imfs.load_snapshot(
"/project/src",
VfsSnapshot::dir([
("a.luau", VfsSnapshot::file("return 1")),
("b.luau", VfsSnapshot::file("return 2")),
]),
)
.unwrap();
let vfs = Vfs::new(imfs);
let snap = snapshot_dir(
&InstanceContext::default(),
&vfs,
Path::new("/project/src"),
"src",
)
.unwrap()
.unwrap();
assert!(
!snap.metadata.ignore_unknown_instances,
"Directory without glob-ignored children should have ignore_unknown_instances = false"
);
assert!(
!snap.metadata.glob_ignored_children,
"Directory without glob-ignored children should have glob_ignored_children = false"
);
assert_eq!(snap.children.len(), 2);
}
}