hakowork/rojo
3
0
Fork 0
mirror of https://github.com/rojo-rbx/rojo.git synced 2026-10-23 14:01:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add prefetch cache for VFS to optimize file reads

Browse source 
- Introduced a `PrefetchCache` structure to store pre-read file contents and canonical paths, enhancing the speed of initial reads during snapshot building.
- Implemented parallel file reading using `rayon` and `walkdir` to populate the cache efficiently.
- Updated the `VfsInner` struct to include the prefetch cache and modified read methods to utilize it, reducing I/O operations.
- Added functions to set and clear the prefetch cache in the `Vfs` API, improving memory management during live operations.
- Enhanced tests to validate the functionality of the prefetch cache, ensuring correct behavior during file reads and canonicalization.

These changes aim to significantly improve the performance of the VFS by minimizing file access times during initial operations.
This commit is contained in:
Joe 2026-02-19 02:04:15 -06:00
parent b105d86541
commit 1ffed1212d
8 changed files with 771 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

64
src/rojo_ref.rs
View file

@ -370,38 +370,38 @@ impl RefPathIndex {
/// absolute using `tree` so the index always stores absolute target paths.
/// This ensures prefix-based lookup works correctly for rename updates.
pub fn populate_from_dir(&mut self, root: &Path, tree: &crate::snapshot::RojoTree) {
let mut count = 0usize;
let mut dirs_to_visit = vec![root.to_path_buf()];
while let Some(dir) = dirs_to_visit.pop() {
let entries = match std::fs::read_dir(&dir) {
Ok(entries) => entries,
Err(_) => continue,
};
for entry in entries.filter_map(|e| e.ok()) {
let path = entry.path();
if path.is_dir() {
dirs_to_visit.push(path);
continue;
}
let name = match path.file_name().and_then(|n| n.to_str()) {
Some(n) => n,
None => continue,
};
if !(name.ends_with(".meta.json5")
|| name.ends_with(".model.json5")
|| name.ends_with(".meta.json")
|| name.ends_with(".model.json"))
{
continue;
}
use rayon::prelude::*;
use walkdir::WalkDir;
fn is_meta_or_model(name: &str) -> bool {
name.ends_with(".meta.json5")
|| name.ends_with(".model.json5")
|| name.ends_with(".meta.json")
|| name.ends_with(".model.json")
}
let meta_paths: Vec<std::path::PathBuf> = WalkDir::new(root)
.follow_links(true)
.into_iter()
.filter_map(|e: Result<walkdir::DirEntry, _>| e.ok())
.filter(|e: &walkdir::DirEntry| {
e.file_type().is_file()
&& e.file_name().to_str().is_some_and(is_meta_or_model)
})
.map(|e: walkdir::DirEntry| e.into_path())
.collect();
let entries: Vec<(String, std::path::PathBuf)> = meta_paths
.par_iter()
.flat_map(|path| {
let source_abs = tree
.get_ids_at_path(&path)
.get_ids_at_path(path)
.first()
.map(|&id| crate::ref_target_path_from_tree(tree, id))
.unwrap_or_default();
if let Ok(bytes) = std::fs::read(&path) {
let mut results = Vec::new();
if let Ok(bytes) = std::fs::read(path) {
if let Ok(val) = crate::json::from_slice::<serde_json::Value>(&bytes) {
if let Some(attrs) = val.get("attributes").and_then(|a| a.as_object()) {
for (key, value) in attrs {
@ -412,16 +412,22 @@ impl RefPathIndex {
&source_abs,
)
.unwrap_or_else(|| path_str.to_string());
self.add(&resolved, &path);
count += 1;
results.push((resolved, path.clone()));
}
}
}
}
}
}
}
results
})
.collect();
let count = entries.len();
for (resolved, path) in entries {
self.add(&resolved, &path);
}
if count > 0 {
log::info!(
"RefPathIndex: populated {} Rojo_Ref_* entries from existing meta/model files",

59
src/serve_session.rs
View file

@ -8,7 +8,7 @@ use std::{
};
use crossbeam_channel::Sender;
use memofs::Vfs;
use memofs::{PrefetchCache, Vfs};
use thiserror::Error;
use crate::{
@ -125,6 +125,43 @@ pub struct ServeSession {
git_repo_root: Option<std::path::PathBuf>,
}
/// Walk a directory tree in parallel, reading all file contents and computing
/// canonical paths. Returns a [`PrefetchCache`] that can be loaded into the
/// VFS to eliminate per-file I/O during the initial snapshot build.
fn prefetch_project_files(root: &Path) -> io::Result<PrefetchCache> {
use rayon::prelude::*;
use std::collections::HashMap;
use walkdir::WalkDir;
let entries: Vec<walkdir::DirEntry> = WalkDir::new(root)
.follow_links(true)
.into_iter()
.filter_map(|e| e.ok())
.collect();
let file_data: Vec<_> = entries
.par_iter()
.filter(|e| e.file_type().is_file())
.filter_map(|e| {
let path = e.path().to_path_buf();
std::fs::read(&path).ok().map(|c| (path, c))
})
.collect();
let canonical_data: Vec<_> = entries
.par_iter()
.filter_map(|e| {
let path = e.path().to_path_buf();
std::fs::canonicalize(&path).ok().map(|c| (path, c))
})
.collect();
Ok(PrefetchCache {
files: file_data.into_iter().collect::<HashMap<_, _>>(),
canonical: canonical_data.into_iter().collect::<HashMap<_, _>>(),
})
}
impl ServeSession {
/// Shared initialization: loads the project and builds the initial
/// snapshot tree. Used by both `new()` and `new_oneshot()`.
@ -133,6 +170,24 @@ impl ServeSession {
let root_project = Project::load_initial_project(vfs, start_path)?;
if std::env::var("ATLAS_SEQUENTIAL").is_err() {
let prefetch_start = Instant::now();
match prefetch_project_files(root_project.folder_location()) {
Ok(cache) => {
log::info!(
"Prefetched {} files + {} canonical paths in {:.1?}",
cache.files.len(),
cache.canonical.len(),
prefetch_start.elapsed()
);
vfs.set_prefetch_cache(cache);
}
Err(err) => {
log::warn!("Prefetch failed, falling back to sequential reads: {err}");
}
}
}
let mut tree = RojoTree::new(InstanceSnapshot::new());
let root_id = tree.get_root_id();
let instance_context = InstanceContext::new();
@ -140,6 +195,8 @@ impl ServeSession {
log::trace!("Generating snapshot of instances from VFS");
let snapshot = snapshot_from_vfs(&instance_context, vfs, start_path)?;
vfs.clear_prefetch_cache();
log::trace!("Computing initial patch set");
let patch_set = compute_patch_set(snapshot, &tree, root_id);

42
src/snapshot_middleware/dir.rs
View file

@ -49,19 +49,41 @@ pub fn snapshot_dir_no_meta(
.all(|rule| rule.passes(child.path()))
};
let mut snapshot_children = Vec::new();
for entry in vfs.read_dir(path)? {
let entry = entry?;
if !passes_filter_rules(&entry) {
continue;
let snapshot_children = if std::env::var("ATLAS_SEQUENTIAL").is_ok() {
let mut children = Vec::new();
for entry in vfs.read_dir(path)? {
let entry = entry?;
if !passes_filter_rules(&entry) {
continue;
}
if let Some(child_snapshot) = snapshot_from_vfs(context, vfs, entry.path())? {
children.push(child_snapshot);
}
}
children
} else {
use rayon::prelude::*;
if let Some(child_snapshot) = snapshot_from_vfs(context, vfs, entry.path())? {
snapshot_children.push(child_snapshot);
let entries: Vec<_> = vfs.read_dir(path)?.filter_map(|e| e.ok()).collect();
let results: Vec<anyhow::Result<Option<InstanceSnapshot>>> = entries
.par_iter()
.map(|entry| {
if !passes_filter_rules(entry) {
return Ok(None);
}
snapshot_from_vfs(context, vfs, entry.path())
})
.collect();
let mut children = Vec::new();
for result in results {
if let Some(snapshot) = result? {
children.push(snapshot);
}
}
}
children
};
let normalized_path = vfs.canonicalize(path)?;
let relevant_paths = vec![