mirror of
https://github.com/rojo-rbx/rojo.git
synced 2026-10-24 15:01:38 +00:00
Implement fix for stem-level deduplication in file handling
- Introduced a migration plan to address the deduplication issue where bare slugs were compared against full filenames, leading to undetected file-format collisions. - Updated `name_for_inst` to return a deduplication key alongside the filename and metadata flag, ensuring callers use the bare slug for collision detection. - Replaced disk-based seeding of `taken_names` with tree-based slugified names to maintain consistency and avoid extension mismatches. - Refactored relevant modules including `dir.rs`, `project.rs`, and `api.rs` to utilize the new deduplication logic. - Updated tests to validate the new deduplication behavior and ensure correct handling of file-format instances.
This commit is contained in:
Joe
parent
126c8e66a4
commit
560fce1352
9 changed files with 333 additions and 208 deletions
|
|
@ -138,24 +138,12 @@ pub fn syncback_dir_no_meta<'sync>(
|
|||
let mut children = Vec::new();
|
||||
let mut removed_children = Vec::new();
|
||||
|
||||
// Seed taken_names with existing files on disk so new children don't
|
||||
// collide with files already present in the directory.
|
||||
let mut taken_names: HashSet<String> = snapshot
|
||||
.vfs()
|
||||
.read_dir(&snapshot.path)
|
||||
.ok()
|
||||
.map(|entries| {
|
||||
entries
|
||||
.filter_map(|e| e.ok())
|
||||
.filter_map(|e| {
|
||||
e.path()
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.map(|n| n.to_lowercase())
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
// taken_names tracks claimed bare slugs (without extensions) for dedup.
|
||||
// It is NOT seeded from disk filenames — those contain extensions that
|
||||
// would cause mismatches with bare-slug comparisons. Instead, old
|
||||
// children's dedup_keys are accumulated as they are processed in the
|
||||
// loop below (via name_for_inst's third return value).
|
||||
let mut taken_names: HashSet<String> = HashSet::new();
|
||||
|
||||
// Detect duplicate child names (case-insensitive for file system safety).
|
||||
// We skip duplicates instead of failing, tracking them in stats.
|
||||
|
|
@ -239,22 +227,18 @@ pub fn syncback_dir_no_meta<'sync>(
|
|||
continue;
|
||||
}
|
||||
// This child exists in both doms. Pass it on.
|
||||
let (child_snap, _needs_meta) = snapshot.with_joined_path(
|
||||
let (child_snap, _needs_meta, dedup_key) = snapshot.with_joined_path(
|
||||
*new_child_ref,
|
||||
Some(old_child.id()),
|
||||
&taken_names,
|
||||
)?;
|
||||
if let Some(name) = child_snap.path.file_name().and_then(|n| n.to_str()) {
|
||||
taken_names.insert(name.to_lowercase());
|
||||
}
|
||||
taken_names.insert(dedup_key.to_lowercase());
|
||||
children.push(child_snap);
|
||||
} else {
|
||||
// The child only exists in the the new dom
|
||||
let (child_snap, _needs_meta) =
|
||||
let (child_snap, _needs_meta, dedup_key) =
|
||||
snapshot.with_joined_path(*new_child_ref, None, &taken_names)?;
|
||||
if let Some(name) = child_snap.path.file_name().and_then(|n| n.to_str()) {
|
||||
taken_names.insert(name.to_lowercase());
|
||||
}
|
||||
taken_names.insert(dedup_key.to_lowercase());
|
||||
children.push(child_snap);
|
||||
}
|
||||
}
|
||||
|
|
@ -299,11 +283,9 @@ pub fn syncback_dir_no_meta<'sync>(
|
|||
if snapshot.should_ignore_tree(*new_child_ref) {
|
||||
continue;
|
||||
}
|
||||
let (child_snap, _needs_meta) =
|
||||
let (child_snap, _needs_meta, dedup_key) =
|
||||
snapshot.with_joined_path(*new_child_ref, None, &taken_names)?;
|
||||
if let Some(name) = child_snap.path.file_name().and_then(|n| n.to_str()) {
|
||||
taken_names.insert(name.to_lowercase());
|
||||
}
|
||||
taken_names.insert(dedup_key.to_lowercase());
|
||||
children.push(child_snap);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,6 +77,11 @@ pub fn syncback_json_model<'sync>(
|
|||
// could have a performance cost.
|
||||
model.schema = old_inst.metadata().schema.clone();
|
||||
model.name = old_inst.metadata().specified_name.clone();
|
||||
} else if snapshot.needs_meta_name {
|
||||
// New instance whose filesystem name differs from the instance name
|
||||
// (due to slugification or deduplication). Store the real name so it
|
||||
// survives a round-trip through the filesystem.
|
||||
model.name = Some(snapshot.new_inst().name.clone());
|
||||
}
|
||||
|
||||
let serialized = match crate::json::to_vec_pretty_sorted(&model) {
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ use crate::{
|
|||
json,
|
||||
resolution::UnresolvedValue,
|
||||
snapshot::InstanceSnapshot,
|
||||
syncback::{name_needs_slugify, SyncbackSnapshot},
|
||||
syncback::SyncbackSnapshot,
|
||||
RojoRef,
|
||||
};
|
||||
|
||||
|
|
@ -165,15 +165,11 @@ impl AdjacentMetadata {
|
|||
.old_inst()
|
||||
.and_then(|inst| inst.metadata().specified_name.clone())
|
||||
.or_else(|| {
|
||||
// If this is a new instance and its name is invalid for the filesystem,
|
||||
// we need to specify the name in meta.json so it can be preserved
|
||||
if snapshot.old_inst().is_none() {
|
||||
let instance_name = &snapshot.new_inst().name;
|
||||
if name_needs_slugify(instance_name) {
|
||||
Some(instance_name.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
// If this is a new instance and its filesystem name differs from
|
||||
// the instance name (due to slugification or deduplication), we
|
||||
// need to specify the name in meta.json so it can be preserved.
|
||||
if snapshot.old_inst().is_none() && snapshot.needs_meta_name {
|
||||
Some(snapshot.new_inst().name.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
|
|
@ -431,15 +427,11 @@ impl DirectoryMetadata {
|
|||
.old_inst()
|
||||
.and_then(|inst| inst.metadata().specified_name.clone())
|
||||
.or_else(|| {
|
||||
// If this is a new instance and its name is invalid for the filesystem,
|
||||
// we need to specify the name in meta.json so it can be preserved
|
||||
if snapshot.old_inst().is_none() {
|
||||
let instance_name = &snapshot.new_inst().name;
|
||||
if name_needs_slugify(instance_name) {
|
||||
Some(instance_name.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
// If this is a new instance and its filesystem name differs from
|
||||
// the instance name (due to slugification or deduplication), we
|
||||
// need to specify the name in meta.json so it can be preserved.
|
||||
if snapshot.old_inst().is_none() && snapshot.needs_meta_name {
|
||||
Some(snapshot.new_inst().name.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
|
|
|
|||
|
|
@ -648,8 +648,8 @@ pub fn syncback_project<'sync>(
|
|||
// All of the children in this loop are by their nature not in the
|
||||
// project, so we just need to run syncback on them.
|
||||
//
|
||||
// Track taken names per parent directory so siblings under the same
|
||||
// parent deduplicate against each other (and against existing files).
|
||||
// Track taken names (bare slugs) per parent directory so siblings
|
||||
// under the same parent deduplicate against each other.
|
||||
let mut taken_names_per_dir: std::collections::HashMap<
|
||||
std::path::PathBuf,
|
||||
std::collections::HashSet<String>,
|
||||
|
|
@ -690,35 +690,14 @@ pub fn syncback_project<'sync>(
|
|||
if parent_middleware != Middleware::Project {
|
||||
let taken_names = taken_names_per_dir
|
||||
.entry(parent_path.clone())
|
||||
.or_insert_with(|| {
|
||||
// Seed with existing files on disk so new children
|
||||
// don't collide with files already present.
|
||||
vfs.read_dir(&parent_path)
|
||||
.ok()
|
||||
.map(|entries| {
|
||||
entries
|
||||
.filter_map(|e| e.ok())
|
||||
.filter_map(|e| {
|
||||
e.path()
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.map(|n| n.to_lowercase())
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
});
|
||||
let (child_snap, _needs_meta) = snapshot.with_base_path(
|
||||
.or_insert_with(std::collections::HashSet::new);
|
||||
let (child_snap, _needs_meta, dedup_key) = snapshot.with_base_path(
|
||||
&parent_path,
|
||||
new_child.referent(),
|
||||
None,
|
||||
taken_names,
|
||||
)?;
|
||||
if let Some(file_name) =
|
||||
child_snap.path.file_name().and_then(|n| n.to_str())
|
||||
{
|
||||
taken_names.insert(file_name.to_lowercase());
|
||||
}
|
||||
taken_names.insert(dedup_key.to_lowercase());
|
||||
descendant_snapshots.push(child_snap);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue