hakowork/rojo
3
0
Fork 0
mirror of https://github.com/rojo-rbx/rojo.git synced 2026-10-23 22:11:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Optimize syncback performance for large projects (~42% faster)

Browse source 
Syncback on a 132K-file project dropped from ~32s to ~18s by:
- Skipping hash_tree computation in clean (non-incremental) mode,
  since hashes are only consulted when incremental=true (-6s)
- Reusing prefetch walkdir results for orphan detection instead of
  walking the filesystem a second time (-1.7s)
- Rewriting collect_all_paths from O(n*d) bottom-up ancestor walks
  to O(n) top-down BFS (-0.7s)
- Adding file-size-first comparison in write_to_vfs_parallel to
  avoid reading files that differ in size
- Scoping git ls-files --stage to project directory prefix
- Adding [PERF] instrumentation at info level for every pipeline phase
This commit is contained in:
Joe 2026-03-07 16:01:51 -06:00
parent 338f97040c
commit cd465a90a2
6 changed files with 425 additions and 225 deletions
Download patch file Download diff file Expand all files Collapse all files

61
src/syncback/fs_snapshot.rs
View file

@ -291,6 +291,16 @@ impl FsSnapshot {
) -> io::Result<()> {
let base_path = base.as_ref();
log::info!(
"[PERF] write_to_vfs_parallel: {} files, {} dirs to add; {} files, {} dirs to remove",
self.added_files.len(),
self.added_dirs.len(),
self.removed_files.len(),
self.removed_dirs.len(),
);
let phase1_timer = std::time::Instant::now();
// Phase 1: Create directories (sequential - parent must exist before child)
let skipped_dirs = AtomicUsize::new(0);
{
@ -308,11 +318,21 @@ impl FsSnapshot {
};
}
} // Release lock before parallel phase
log::info!(
"[PERF] phase1 create dirs: {:.3}s",
phase1_timer.elapsed().as_secs_f64()
);
let phase2_timer = std::time::Instant::now();
// Phase 2: Write files (parallel - independent operations)
let write_errors = AtomicUsize::new(0);
let first_error: std::sync::Mutex<Option<io::Error>> = std::sync::Mutex::new(None);
let skipped_files = AtomicUsize::new(0);
let git_skipped = AtomicUsize::new(0);
let byte_skipped = AtomicUsize::new(0);
let size_skipped = AtomicUsize::new(0);
self.added_files.par_iter().for_each(|(path, contents)| {
let full_path = base_path.join(path);
@ -320,16 +340,24 @@ impl FsSnapshot {
if let Some(cache) = git_cache {
if cache.file_matches_index(path, contents) {
skipped_files.fetch_add(1, Ordering::Relaxed);
git_skipped.fetch_add(1, Ordering::Relaxed);
return;
}
}
match std::fs::read(&full_path) {
Ok(existing) if existing == *contents => {
skipped_files.fetch_add(1, Ordering::Relaxed);
return;
match std::fs::metadata(&full_path) {
Ok(meta) if meta.len() != contents.len() as u64 => {
size_skipped.fetch_add(1, Ordering::Relaxed);
}
_ => {}
Ok(_) => match std::fs::read(&full_path) {
Ok(existing) if existing == *contents => {
skipped_files.fetch_add(1, Ordering::Relaxed);
byte_skipped.fetch_add(1, Ordering::Relaxed);
return;
}
_ => {}
},
Err(_) => {}
}
if let Err(err) = write_with_retry(&full_path, contents) {
@ -341,6 +369,14 @@ impl FsSnapshot {
}
});
log::info!(
"[PERF] phase2 write files: {:.3}s (git_skip={}, byte_skip={}, size_diff={})",
phase2_timer.elapsed().as_secs_f64(),
git_skipped.load(Ordering::Relaxed),
byte_skipped.load(Ordering::Relaxed),
size_skipped.load(Ordering::Relaxed),
);
// Check for write errors
if let Some(err) = first_error.into_inner().unwrap() {
let error_count = write_errors.load(Ordering::Relaxed);
@ -350,8 +386,9 @@ impl FsSnapshot {
return Err(err);
}
let phase3_timer = std::time::Instant::now();
// Phase 3: Remove files not inside removed directories (parallel)
// Uses retry logic on Windows for transient errors.
let files_to_remove: Vec<_> = self
.removed_files
.iter()
@ -372,6 +409,11 @@ impl FsSnapshot {
}
});
log::info!(
"[PERF] phase3 remove files: {:.3}s",
phase3_timer.elapsed().as_secs_f64()
);
// Check for remove errors
if let Some(err) = first_remove_error.into_inner().unwrap() {
let error_count = remove_errors.load(Ordering::Relaxed);
@ -384,6 +426,8 @@ impl FsSnapshot {
return Err(err);
}
let phase4_timer = std::time::Instant::now();
// Phase 4: Remove directories (sequential - uses VFS for proper unwatch handling)
{
let mut lock = vfs.lock();
@ -402,6 +446,11 @@ impl FsSnapshot {
}
}
log::info!(
"[PERF] phase4 remove dirs: {:.3}s",
phase4_timer.elapsed().as_secs_f64()
);
let skipped_file_count = skipped_files.load(Ordering::Relaxed);
let skipped_dir_count = skipped_dirs.load(Ordering::Relaxed);
let written_files = self.added_files.len() - skipped_file_count;

412
src/syncback/mod.rs
View file

@ -97,11 +97,34 @@ pub fn syncback_loop(
project: &Project,
incremental: bool,
) -> anyhow::Result<SyncbackResult> {
syncback_loop_with_stats(vfs, old_tree, new_tree, project, incremental, None)
syncback_loop_with_stats(vfs, old_tree, new_tree, project, incremental, None, None)
}
pub fn syncback_loop_with_walked_paths(
vfs: &Vfs,
old_tree: &mut RojoTree,
new_tree: WeakDom,
project: &Project,
incremental: bool,
pre_walked_paths: Option<HashSet<PathBuf>>,
) -> anyhow::Result<SyncbackResult> {
syncback_loop_with_stats(
vfs,
old_tree,
new_tree,
project,
incremental,
None,
pre_walked_paths,
)
}
/// Runs the syncback loop with an optional external stats tracker.
/// If no stats are provided, an internal one is created and logged at the end.
///
/// `pre_walked_paths`: If provided, these paths are used for orphan detection
/// instead of re-walking the filesystem. Avoids a redundant walkdir when the
/// caller has already enumerated the project files (e.g. via prefetch).
pub fn syncback_loop_with_stats(
vfs: &Vfs,
old_tree: &mut RojoTree,
@ -109,6 +132,7 @@ pub fn syncback_loop_with_stats(
project: &Project,
incremental: bool,
external_stats: Option<&SyncbackStats>,
pre_walked_paths: Option<HashSet<PathBuf>>,
) -> anyhow::Result<SyncbackResult> {
// Create internal stats if not provided externally
let internal_stats = SyncbackStats::new();
@ -130,11 +154,11 @@ pub fn syncback_loop_with_stats(
// Collect all instance paths BEFORE pruning so we can track external references
// (references to instances that will be pruned, like SoundGroups in SoundService).
log::debug!("Collecting instance paths before pruning...");
log::info!("[PERF] syncback_loop entered");
let pre_prune_paths = collect_all_paths(&new_tree);
log::debug!(
"syncback: collect_all_paths in {:.1?}",
phase_timer.elapsed()
log::info!(
"[PERF] collect_all_paths: {:.3}s",
phase_timer.elapsed().as_secs_f64()
);
// TODO: Add a better way to tell if the root of a project is a directory
@ -171,14 +195,17 @@ pub fn syncback_loop_with_stats(
if ignore_hidden {
strip_hidden_services(&mut new_tree);
}
log::debug!("syncback: prune + filter in {:.1?}", phase_timer.elapsed());
log::info!(
"[PERF] prune + filter: {:.3}s",
phase_timer.elapsed().as_secs_f64()
);
let phase_timer = std::time::Instant::now();
let mut deferred_referents = collect_referents(&new_tree, &pre_prune_paths, None);
let placeholder_map = std::mem::take(&mut deferred_referents.placeholder_to_source_and_target);
log::debug!(
"syncback: collect_referents in {:.1?}",
phase_timer.elapsed()
log::info!(
"[PERF] collect_referents: {:.3}s",
phase_timer.elapsed().as_secs_f64()
);
let phase_timer = std::time::Instant::now();
@ -235,211 +262,206 @@ pub fn syncback_loop_with_stats(
if !ignore_referents {
link_referents(deferred_referents, &mut new_tree)?;
}
log::debug!(
"syncback: filter props + link refs in {:.1?}",
phase_timer.elapsed()
log::info!(
"[PERF] filter props + link refs: {:.3}s",
phase_timer.elapsed().as_secs_f64()
);
new_tree.root_mut().name = old_tree.root().name().to_string();
let phase_timer = std::time::Instant::now();
let (old_hashes, new_hashes) = rayon::join(
|| hash_tree(project, old_tree.inner(), old_tree.get_root_id()),
|| hash_tree(project, &new_tree, new_tree.root_ref()),
);
log::debug!(
"syncback: hash both trees (parallel) in {:.1?}",
phase_timer.elapsed()
);
let (old_hashes, new_hashes) = if incremental {
let result = rayon::join(
|| hash_tree(project, old_tree.inner(), old_tree.get_root_id()),
|| hash_tree(project, &new_tree, new_tree.root_ref()),
);
log::info!(
"[PERF] hash both trees (parallel): {:.3}s",
phase_timer.elapsed().as_secs_f64()
);
result
} else {
log::info!("[PERF] hash skipped (clean mode)");
(HashMap::new(), HashMap::new())
};
let project_path = project.folder_location();
let phase_timer = std::time::Instant::now();
let existing_paths: HashSet<PathBuf> = if !incremental {
let mut paths = HashSet::new();
if let Some(pre_walked) = pre_walked_paths.filter(|p| p.len() > 100) {
log::info!(
"[PERF] orphan scan: reusing {} pre-walked paths",
pre_walked.len()
);
pre_walked
} else {
let mut paths = HashSet::new();
// Get the source directories from the project's tree structure.
// We need to collect ALL $path directories defined in the project,
// not just instigating_source metadata (which may point to project file).
let mut dirs_to_scan: Vec<PathBuf> = Vec::new();
// Get the source directories from the project's tree structure.
// We need to collect ALL $path directories defined in the project,
// not just instigating_source metadata (which may point to project file).
let mut dirs_to_scan: Vec<PathBuf> = Vec::new();
// Helper to recursively collect $path DIRECTORY entries from project tree
// Only directories need to be scanned for orphan detection; single files
// don't contain orphans that need to be removed.
fn collect_paths_from_project(
node: &crate::project::ProjectNode,
base_path: &Path,
paths: &mut Vec<PathBuf>,
) {
// If this node has a $path that points to a directory, add it
if let Some(ref path_node) = node.path {
let resolved = base_path.join(path_node.path());
// Only add directories - single files don't need orphan scanning
if resolved.is_dir() && !paths.contains(&resolved) {
log::trace!("Found $path directory in project: {}", resolved.display());
paths.push(resolved);
// Recursively collect $path DIRECTORY entries from project tree.
// Only directories need to be scanned for orphan detection.
fn collect_paths_from_project(
node: &crate::project::ProjectNode,
base_path: &Path,
paths: &mut Vec<PathBuf>,
) {
if let Some(ref path_node) = node.path {
let resolved = base_path.join(path_node.path());
if resolved.is_dir() && !paths.contains(&resolved) {
log::trace!("Found $path directory in project: {}", resolved.display());
paths.push(resolved);
}
}
for child in node.children.values() {
collect_paths_from_project(child, base_path, paths);
}
}
// Recursively check children
for child in node.children.values() {
collect_paths_from_project(child, base_path, paths);
}
}
// Collect paths from the project tree definition
collect_paths_from_project(&project.tree, project_path, &mut dirs_to_scan);
collect_paths_from_project(&project.tree, project_path, &mut dirs_to_scan);
// NOTE: We intentionally do NOT scan the project root folder itself.
// Only directories explicitly referenced via $path should be scanned.
// Scanning the root would delete unrelated files like .gitignore, README.md, etc.
//
// However, we DO need to collect alternate file representations of $path entries.
// For example, if $path: "src" expects a directory, but "src.luau" exists as a file
// (perhaps the user accidentally converted the directory to a file), that orphan
// file should be removed. We collect these as individual files to check, not as
// directories to scan recursively.
let mut orphan_files_to_check: Vec<PathBuf> = Vec::new();
// Collect alternate file representations of $path entries.
// e.g., if $path: "src" expects a directory, but "src.luau" exists as a file,
// that orphan file should be removed.
let mut orphan_files_to_check: Vec<PathBuf> = Vec::new();
// Helper to find alternate file extensions for a path
fn collect_alternate_files(
node: &crate::project::ProjectNode,
base_path: &Path,
files: &mut Vec<PathBuf>,
) {
if let Some(ref path_node) = node.path {
let path_str = path_node.path();
let resolved = base_path.join(path_str);
fn collect_alternate_files(
node: &crate::project::ProjectNode,
base_path: &Path,
files: &mut Vec<PathBuf>,
) {
if let Some(ref path_node) = node.path {
let path_str = path_node.path();
let resolved = base_path.join(path_str);
// If the $path points to a directory, check for file alternates
if resolved.is_dir() {
// Common Rojo file extensions that could be alternates
let extensions = [
".luau",
".lua",
".server.luau",
".server.lua",
".client.luau",
".client.lua",
".local.luau",
".local.lua",
];
for ext in extensions {
// Create alternate filename by appending extension to path name
let path_str_display = path_str.display().to_string();
let alt_file = base_path.join(format!("{}{}", path_str_display, ext));
if alt_file.exists() && alt_file.is_file() {
log::trace!(
"Found alternate file for $path '{}': {}",
path_str_display,
alt_file.display()
);
files.push(alt_file);
if resolved.is_dir() {
let extensions = [
".luau",
".lua",
".server.luau",
".server.lua",
".client.luau",
".client.lua",
".local.luau",
".local.lua",
];
for ext in extensions {
let path_str_display = path_str.display().to_string();
let alt_file = base_path.join(format!("{}{}", path_str_display, ext));
if alt_file.exists() && alt_file.is_file() {
log::trace!(
"Found alternate file for $path '{}': {}",
path_str_display,
alt_file.display()
);
files.push(alt_file);
}
}
}
}
}
for child in node.children.values() {
collect_alternate_files(child, base_path, files);
}
}
collect_alternate_files(&project.tree, project_path, &mut orphan_files_to_check);
// Also check instance metadata for paths that might not be in project
// (e.g., from nested project references)
let root = old_tree.root();
log::trace!(
"Root instance: name={}, class={}",
root.name(),
root.class_name()
);
if let Some(source) = &root.metadata().instigating_source {
let path = source.path();
// Skip project files - we want source directories
if !path.to_string_lossy().ends_with(".project.json5")
&& !path.to_string_lossy().ends_with(".project.json")
{
log::trace!("Root has instigating_source: {}", path.display());
if path.exists() && !dirs_to_scan.contains(&path.to_path_buf()) {
dirs_to_scan.push(path.to_path_buf());
for child in node.children.values() {
collect_alternate_files(child, base_path, files);
}
}
}
collect_alternate_files(&project.tree, project_path, &mut orphan_files_to_check);
// Check children for additional paths
for ref_id in old_tree.inner().root().children() {
if let Some(inst) = old_tree.get_instance(*ref_id) {
if let Some(source) = &inst.metadata().instigating_source {
let path = source.path();
if !path.to_string_lossy().ends_with(".project.json5")
&& !path.to_string_lossy().ends_with(".project.json")
&& path.exists()
&& !dirs_to_scan.contains(&path.to_path_buf())
{
// Check instance metadata for paths that might not be in the project
// (e.g., from nested project references)
let root = old_tree.root();
log::trace!(
"Root instance: name={}, class={}",
root.name(),
root.class_name()
);
if let Some(source) = &root.metadata().instigating_source {
let path = source.path();
if !path.to_string_lossy().ends_with(".project.json5")
&& !path.to_string_lossy().ends_with(".project.json")
{
log::trace!("Root has instigating_source: {}", path.display());
if path.exists() && !dirs_to_scan.contains(&path.to_path_buf()) {
dirs_to_scan.push(path.to_path_buf());
}
}
}
}
if log::log_enabled!(log::Level::Trace) {
let dirs_str: Vec<_> = dirs_to_scan
.iter()
.map(|p| p.display().to_string())
.collect();
log::trace!("dirs_to_scan: {:?}", dirs_str);
}
for dir in &dirs_to_scan {
if !dir.is_dir() {
continue;
}
if let Some(name) = dir.file_name().and_then(|n| n.to_str()) {
if name.starts_with('.') && name != ".gitkeep" {
continue;
}
}
for entry in walkdir::WalkDir::new(dir)
.follow_links(true)
.into_iter()
.filter_entry(|e| {
if e.depth() == 0 {
return true;
for ref_id in old_tree.inner().root().children() {
if let Some(inst) = old_tree.get_instance(*ref_id) {
if let Some(source) = &inst.metadata().instigating_source {
let path = source.path();
if !path.to_string_lossy().ends_with(".project.json5")
&& !path.to_string_lossy().ends_with(".project.json")
&& path.exists()
&& !dirs_to_scan.contains(&path.to_path_buf())
{
dirs_to_scan.push(path.to_path_buf());
}
}
e.file_name()
.to_str()
.is_none_or(|n| !n.starts_with('.') || n == ".gitkeep")
})
.flatten()
{
if entry.depth() == 0 {
}
}
if log::log_enabled!(log::Level::Trace) {
let dirs_str: Vec<_> = dirs_to_scan
.iter()
.map(|p| p.display().to_string())
.collect();
log::trace!("dirs_to_scan: {:?}", dirs_str);
}
for dir in &dirs_to_scan {
if !dir.is_dir() {
continue;
}
let path = entry.path().to_path_buf();
if !is_valid_path(&ignore_patterns, project_path, &path) {
if let Some(name) = dir.file_name().and_then(|n| n.to_str()) {
if name.starts_with('.') && name != ".gitkeep" {
continue;
}
}
for entry in walkdir::WalkDir::new(dir)
.follow_links(true)
.into_iter()
.filter_entry(|e| {
if e.depth() == 0 {
return true;
}
e.file_name()
.to_str()
.is_none_or(|n| !n.starts_with('.') || n == ".gitkeep")
})
.flatten()
{
if entry.depth() == 0 {
continue;
}
let path = entry.path().to_path_buf();
if !is_valid_path(&ignore_patterns, project_path, &path) {
continue;
}
paths.insert(path);
}
}
for file in &orphan_files_to_check {
if !is_valid_path(&ignore_patterns, project_path, file) {
continue;
}
paths.insert(path);
log::trace!("Adding alternate file to orphan check: {}", file.display());
paths.insert(file.clone());
}
}
// Also add any alternate file representations we found earlier
// (e.g., src.luau when $path: "src" is a directory)
for file in &orphan_files_to_check {
if !is_valid_path(&ignore_patterns, project_path, file) {
continue;
}
log::trace!("Adding alternate file to orphan check: {}", file.display());
paths.insert(file.clone());
}
log::debug!("Scanned {} existing paths from filesystem", paths.len());
paths
log::debug!("Scanned {} existing paths from filesystem", paths.len());
paths
} // end else (no pre_walked_paths)
} else {
HashSet::new()
};
log::debug!(
"syncback: orphan scan in {:.1?} ({} paths)",
phase_timer.elapsed(),
log::info!(
"[PERF] orphan scan: {:.3}s ({} paths)",
phase_timer.elapsed().as_secs_f64(),
existing_paths.len()
);
@ -468,9 +490,16 @@ pub fn syncback_loop_with_stats(
let mut fs_snapshot = FsSnapshot::new();
let mut instance_paths: HashMap<Ref, Vec<PathBuf>> = HashMap::new();
let mut walk_count: usize = 0;
let mut walk_syncback_ns: u128 = 0;
let mut walk_merge_ns: u128 = 0;
let mut walk_path_ns: u128 = 0;
'syncback: while let Some(snapshot) = snapshots.pop() {
walk_count += 1;
let path_start = std::time::Instant::now();
let inst_path = snapshot.get_new_inst_path(snapshot.new);
walk_path_ns += path_start.elapsed().as_nanos();
// In incremental mode, we can quickly check that two subtrees are identical
// and if they are, skip reconciling them. In clean mode, we always process
// all instances to ensure fresh structure.
@ -517,6 +546,7 @@ pub fn syncback_loop_with_stats(
continue;
}
let syncback_start = std::time::Instant::now();
let syncback = match middleware.syncback(&snapshot) {
Ok(syncback) => syncback,
Err(err) if middleware == Middleware::Dir => {
@ -551,6 +581,8 @@ pub fn syncback_loop_with_stats(
Err(err) => anyhow::bail!("Failed to syncback {inst_path} because {err}"),
};
walk_syncback_ns += syncback_start.elapsed().as_nanos();
if !syncback.removed_children.is_empty() {
log::debug!(
"removed children for {inst_path}: {}",
@ -603,13 +635,22 @@ pub fn syncback_loop_with_stats(
instance_paths.insert(snapshot.new, files_for_instance);
}
let merge_start = std::time::Instant::now();
fs_snapshot.merge_with_filter(syncback.fs_snapshot, |path| {
is_valid_path(&ignore_patterns, project_path, path)
});
walk_merge_ns += merge_start.elapsed().as_nanos();
snapshots.extend(syncback.children);
}
log::debug!("syncback: main walk loop in {:.1?}", phase_timer.elapsed());
log::info!(
"[PERF] main walk loop: {:.3}s ({} instances, syncback={:.3}s, merge={:.3}s, path={:.3}s)",
phase_timer.elapsed().as_secs_f64(),
walk_count,
walk_syncback_ns as f64 / 1e9,
walk_merge_ns as f64 / 1e9,
walk_path_ns as f64 / 1e9,
);
let phase_timer = std::time::Instant::now();
{
@ -629,19 +670,19 @@ pub fn syncback_loop_with_stats(
let relative = crate::compute_relative_ref_path(&source_abs, &target_abs);
substitutions.push((placeholder.clone(), relative));
}
log::debug!(
"syncback: built {} ref substitutions in {:.1?}",
substitutions.len(),
phase_timer.elapsed()
log::info!(
"[PERF] build ref substitutions: {:.3}s (count={})",
phase_timer.elapsed().as_secs_f64(),
substitutions.len()
);
let sub_timer = std::time::Instant::now();
if !substitutions.is_empty() {
fs_snapshot.fix_ref_paths(&substitutions);
}
log::debug!(
"syncback: applied ref substitutions in {:.1?}",
sub_timer.elapsed()
log::info!(
"[PERF] apply ref substitutions: {:.3}s",
sub_timer.elapsed().as_secs_f64()
);
}
@ -870,7 +911,10 @@ pub fn syncback_loop_with_stats(
}
}
log::debug!("syncback: orphan removal in {:.1?}", phase_timer.elapsed());
log::info!(
"[PERF] orphan removal: {:.3}s",
phase_timer.elapsed().as_secs_f64()
);
if external_stats.is_none() {
stats.log_summary();

20
src/syncback/ref_properties.rs
View file

@ -172,14 +172,28 @@ struct IdRefLink {
/// Collects all instance paths in a WeakDom before any pruning occurs.
/// Returns a map of Ref -> filesystem-name-compatible path for all instances.
pub fn collect_all_paths(dom: &WeakDom) -> HashMap<Ref, String> {
let root_ref = dom.root_ref();
let mut paths = HashMap::new();
let mut queue = VecDeque::new();
queue.push_back(dom.root_ref());
paths.insert(root_ref, String::new());
queue.push_back(root_ref);
while let Some(inst_ref) = queue.pop_front() {
let inst = dom.get_by_ref(inst_ref).unwrap();
queue.extend(inst.children().iter().copied());
paths.insert(inst_ref, tentative_fs_path(dom, inst_ref));
let parent_path = paths.get(&inst_ref).unwrap().clone();
for &child_ref in inst.children() {
let child = dom.get_by_ref(child_ref).unwrap();
let segment = tentative_fs_name(child);
let child_path = if parent_path.is_empty() {
segment
} else {
format!("{}/{}", parent_path, segment)
};
paths.insert(child_ref, child_path);
queue.push_back(child_ref);
}
}
paths