rojo/src/syncback/mod.rs

mod file_names;
mod fs_snapshot;
mod hash;
mod property_filter;
mod ref_properties;
mod snapshot;
mod stats;

use anyhow::Context;
use indexmap::IndexMap;
use memofs::Vfs;
use rbx_dom_weak::{
    types::{Ref, Variant},
    ustr, Instance, Ustr, UstrSet, WeakDom,
};
use serde::{Deserialize, Serialize};
use std::{
    collections::{HashMap, HashSet, VecDeque},
    env,
    path::{Path, PathBuf},
    sync::OnceLock,
};

use crate::{
    glob::Glob,
    snapshot::{InstanceWithMeta, RojoTree},
    snapshot_middleware::Middleware,
    syncback::ref_properties::{collect_all_paths, collect_referents, link_referents},
    Project,
};

pub use file_names::{extension_for_middleware, name_for_inst, validate_file_name};
pub use fs_snapshot::FsSnapshot;
pub use hash::*;
pub use property_filter::{
    filter_properties, filter_properties_preallocated, should_property_serialize,
};
pub use snapshot::{inst_path, SyncbackData, SyncbackSnapshot};
pub use stats::SyncbackStats;

/// The name of an enviroment variable to use to override the behavior of
/// syncback on model files.
/// By default, syncback will use `Rbxm` for model files.
/// If this is set to `1`, it will instead use `Rbxmx`. If it is set to `2`,
/// it will use `JsonModel`.
///
/// This will **not** override existing `Rbxm` middleware. It will only impact
/// new files.
const DEBUG_MODEL_FORMAT_VAR: &str = "ROJO_SYNCBACK_DEBUG";

/// Services that are considered "visible" and will be included when
/// `ignoreHiddenServices` is enabled. All other services will be ignored.
pub const VISIBLE_SERVICES: &[&str] = &[
    "Lighting",
    "MaterialService",
    "NetworkClient",
    "ReplicatedFirst",
    "ReplicatedStorage",
    "ServerScriptService",
    "ServerStorage",
    "SoundService",
    "StarterGui",
    "StarterPack",
    "StarterPlayer",
    "Teams",
    "TextChatService",
    "VoiceChatService",
    "Workspace",
];

/// A glob that can be used to tell if a path contains a `.git` folder.
static GIT_IGNORE_GLOB: OnceLock<Glob> = OnceLock::new();

pub fn syncback_loop(
    vfs: &Vfs,
    old_tree: &mut RojoTree,
    new_tree: WeakDom,
    project: &Project,
    incremental: bool,
) -> anyhow::Result<FsSnapshot> {
    syncback_loop_with_stats(vfs, old_tree, new_tree, project, incremental, None)
}

/// Runs the syncback loop with an optional external stats tracker.
/// If no stats are provided, an internal one is created and logged at the end.
pub fn syncback_loop_with_stats(
    vfs: &Vfs,
    old_tree: &mut RojoTree,
    mut new_tree: WeakDom,
    project: &Project,
    incremental: bool,
    external_stats: Option<&SyncbackStats>,
) -> anyhow::Result<FsSnapshot> {
    // Create internal stats if not provided externally
    let internal_stats = SyncbackStats::new();
    let stats = external_stats.unwrap_or(&internal_stats);

    let ignore_patterns = project
        .syncback_rules
        .as_ref()
        .map(|rules| rules.compile_globs())
        .transpose()?;

    let tree_globs = project
        .syncback_rules
        .as_ref()
        .map(|rules| rules.compile_tree_globs())
        .transpose()?;

    // Collect all instance paths BEFORE pruning so we can track external references
    // (references to instances that will be pruned, like SoundGroups in SoundService).
    log::debug!("Collecting instance paths before pruning...");
    let pre_prune_paths = collect_all_paths(&new_tree);

    // TODO: Add a better way to tell if the root of a project is a directory
    let skip_pruning = if let Some(path) = &project.tree.path {
        let middleware =
            Middleware::middleware_for_path(vfs, &project.sync_rules, path.path()).unwrap();
        if let Some(middleware) = middleware {
            middleware.is_dir()
        } else {
            false
        }
    } else {
        false
    };
    if !skip_pruning {
        // Strip out any objects from the new tree that aren't in the old tree. This
        // is necessary so that hashing the roots of each tree won't always result
        // in different hashes. Shout out to Roblox for serializing a bunch of
        // Services nobody cares about.
        log::debug!("Pruning new tree");
        strip_unknown_root_children(&mut new_tree, old_tree);
    }

    // If ignoreHiddenServices is enabled (default: true), filter to only visible services
    // Check root-level setting first, then fall back to syncbackRules for backward compatibility
    let ignore_hidden = project
        .ignore_hidden_services
        .or_else(|| {
            project
                .syncback_rules
                .as_ref()
                .map(|rules| rules.ignore_hidden_services())
        })
        .unwrap_or(true);
    if ignore_hidden {
        log::debug!("Filtering hidden services");
        strip_hidden_services(&mut new_tree);
    }

    log::debug!("Collecting referents for new DOM...");
    let deferred_referents = collect_referents(&new_tree, &pre_prune_paths);

    // Remove any properties that are manually blocked from syncback via the
    // project file.
    log::debug!("Pre-filtering properties on DOMs");
    for referent in descendants(&new_tree, new_tree.root_ref()) {
        let new_inst = new_tree.get_by_ref_mut(referent).unwrap();
        if let Some(filter) = get_property_filter(project, new_inst) {
            for prop in filter {
                new_inst.properties.remove(&prop);
            }
        }
    }
    for referent in descendants(old_tree.inner(), old_tree.get_root_id()) {
        let mut old_inst_rojo = old_tree.get_instance_mut(referent).unwrap();
        let old_inst = old_inst_rojo.inner_mut();
        if let Some(filter) = get_property_filter(project, old_inst) {
            for prop in filter {
                old_inst.properties.remove(&prop);
            }
        }
    }

    // Handle removing the current camera.
    if let Some(syncback_rules) = &project.syncback_rules {
        if !syncback_rules.sync_current_camera.unwrap_or_default() {
            log::debug!("Removing CurrentCamera from new DOM");
            let mut camera_ref = None;
            for child_ref in new_tree.root().children() {
                let inst = new_tree.get_by_ref(*child_ref).unwrap();
                if inst.class == "Workspace" {
                    camera_ref = inst.properties.get(&ustr("CurrentCamera"));
                    break;
                }
            }
            if let Some(Variant::Ref(camera_ref)) = camera_ref {
                if new_tree.get_by_ref(*camera_ref).is_some() {
                    new_tree.destroy(*camera_ref);
                }
            }
        }
    }

    let ignore_referents = project
        .syncback_rules
        .as_ref()
        .and_then(|s| s.ignore_referents)
        .unwrap_or_default();
    if !ignore_referents {
        log::debug!("Linking referents for new DOM");
        link_referents(deferred_referents, &mut new_tree)?;
    } else {
        log::debug!("Skipping referent linking as per project syncback rules");
    }

    // As with pruning the children of the new root, we need to ensure the roots
    // for both DOMs have the same name otherwise their hashes will always be
    // different.
    new_tree.root_mut().name = old_tree.root().name().to_string();

    log::debug!("Hashing project DOM");
    let old_hashes = hash_tree(project, old_tree.inner(), old_tree.get_root_id());
    log::debug!("Hashing file DOM");
    let new_hashes = hash_tree(project, &new_tree, new_tree.root_ref());

    let project_path = project.folder_location();

    // In clean mode, we scan the actual filesystem to find ALL existing files,
    // not just what Rojo loaded into its tree. This ensures orphaned files
    // (like duplicates Rojo couldn't load) get cleaned up properly.
    // Clean mode is essentially: "delete everything and rebuild from Studio"
    let existing_paths: HashSet<PathBuf> = if !incremental {
        log::debug!("Clean mode: scanning filesystem for existing paths");
        let mut paths = HashSet::new();

        // Get the source directories from the project's tree structure.
        // We need to collect ALL $path directories defined in the project,
        // not just instigating_source metadata (which may point to project file).
        let mut dirs_to_scan: Vec<PathBuf> = Vec::new();
        
        // Helper to recursively collect $path DIRECTORY entries from project tree
        // Only directories need to be scanned for orphan detection; single files
        // don't contain orphans that need to be removed.
        fn collect_paths_from_project(
            node: &crate::project::ProjectNode,
            base_path: &Path,
            paths: &mut Vec<PathBuf>,
        ) {
            // If this node has a $path that points to a directory, add it
            if let Some(ref path_node) = node.path {
                let resolved = base_path.join(path_node.path());
                // Only add directories - single files don't need orphan scanning
                if resolved.is_dir() && !paths.contains(&resolved) {
                    log::trace!("Found $path directory in project: {}", resolved.display());
                    paths.push(resolved);
                }
            }
            // Recursively check children
            for child in node.children.values() {
                collect_paths_from_project(child, base_path, paths);
            }
        }
        
        // Collect paths from the project tree definition
        collect_paths_from_project(&project.tree, project_path, &mut dirs_to_scan);
        
        // Also scan the project folder itself - files like src.luau that exist
        // alongside default.project.json5 should be considered for orphan removal
        let project_path_buf = project_path.to_path_buf();
        if !dirs_to_scan.contains(&project_path_buf) {
            log::trace!("Adding project folder to scan: {}", project_path.display());
            dirs_to_scan.push(project_path_buf);
        }
        
        // Also check instance metadata for paths that might not be in project
        // (e.g., from nested project references)
        let root = old_tree.root();
        log::trace!("Root instance: name={}, class={}", root.name(), root.class_name());
        if let Some(source) = &root.metadata().instigating_source {
            let path = source.path();
            // Skip project files - we want source directories
            if !path.to_string_lossy().ends_with(".project.json5")
                && !path.to_string_lossy().ends_with(".project.json")
            {
                log::trace!("Root has instigating_source: {}", path.display());
                if path.exists() && !dirs_to_scan.contains(&path.to_path_buf()) {
                    dirs_to_scan.push(path.to_path_buf());
                }
            }
        }
        
        // Check children for additional paths
        for ref_id in old_tree.inner().root().children() {
            if let Some(inst) = old_tree.get_instance(*ref_id) {
                if let Some(source) = &inst.metadata().instigating_source {
                    let path = source.path();
                    if !path.to_string_lossy().ends_with(".project.json5")
                        && !path.to_string_lossy().ends_with(".project.json")
                    {
                        if path.exists() && !dirs_to_scan.contains(&path.to_path_buf()) {
                            dirs_to_scan.push(path.to_path_buf());
                        }
                    }
                }
            }
        }

        // Recursively scan each source directory
        fn scan_directory(
            dir: &Path,
            paths: &mut HashSet<PathBuf>,
            ignore_patterns: &Option<Vec<Glob>>,
            project_path: &Path,
        ) {
            log::trace!("Scanning directory: {}", dir.display());
            let entries = match std::fs::read_dir(dir) {
                Ok(e) => e,
                Err(e) => {
                    log::trace!("Failed to read directory {}: {}", dir.display(), e);
                    return;
                }
            };

            for entry in entries.flatten() {
                let path = entry.path();

                // Skip paths matching ignore patterns
                if !is_valid_path(ignore_patterns, project_path, &path) {
                    log::trace!("Skipping {} (matches ignore pattern)", path.display());
                    continue;
                }

                // Skip hidden files/directories (starting with .)
                if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
                    if name.starts_with('.') {
                        log::trace!("Skipping {} (hidden)", path.display());
                        continue;
                    }
                }

                log::trace!("Found path: {}", path.display());
                paths.insert(path.clone());

                if path.is_dir() {
                    scan_directory(&path, paths, ignore_patterns, project_path);
                }
            }
        }

        log::trace!(
            "dirs_to_scan: {:?}",
            dirs_to_scan.iter().map(|p| p.display().to_string()).collect::<Vec<_>>()
        );

        for dir in &dirs_to_scan {
            if dir.is_dir() {
                scan_directory(dir, &mut paths, &ignore_patterns, project_path);
            }
            // Note: We only add directories to dirs_to_scan now, so no else branch needed
        }

        log::debug!("Scanned {} existing paths from filesystem", paths.len());
        paths
    } else {
        HashSet::new()
    };

    let syncback_data = SyncbackData {
        vfs,
        old_tree,
        new_tree: &new_tree,
        project,
        incremental,
        stats,
    };

    // Always start with old reference for the Project middleware.
    // In clean mode, child snapshots will have old=None (handled by SyncbackSnapshot methods).
    let mut snapshots = vec![SyncbackSnapshot {
        data: syncback_data,
        old: Some(old_tree.get_root_id()),
        new: new_tree.root_ref(),
        path: project.file_location.clone(),
        middleware: Some(Middleware::Project),
    }];

    let mut fs_snapshot = FsSnapshot::new();

    'syncback: while let Some(snapshot) = snapshots.pop() {
        let inst_path = snapshot.get_new_inst_path(snapshot.new);
        // In incremental mode, we can quickly check that two subtrees are identical
        // and if they are, skip reconciling them. In clean mode, we always process
        // all instances to ensure fresh structure.
        if incremental {
            if let Some(old_ref) = snapshot.old {
                match (old_hashes.get(&old_ref), new_hashes.get(&snapshot.new)) {
                    (Some(old), Some(new)) => {
                        if old == new {
                            log::trace!(
                                "Skipping {inst_path} due to it being identically hashed as {old:?}"
                            );
                            continue;
                        }
                    }
                    _ => unreachable!("All Instances in both DOMs should have hashes"),
                }
            }
        }

        if !is_valid_path(&ignore_patterns, project_path, &snapshot.path) {
            log::debug!("Skipping {inst_path} because its path matches ignore pattern");
            continue;
        }
        // Check ignoreTrees with glob pattern support
        if let Some(ref globs) = tree_globs {
            for glob in globs {
                if glob.is_match(&inst_path) {
                    log::debug!("Tree {inst_path} is blocked by ignoreTrees glob pattern");
                    continue 'syncback;
                }
            }
        }

        let middleware = get_best_middleware(&snapshot);

        log::trace!(
            "Middleware for {inst_path} is {:?} (path is {})",
            middleware,
            snapshot.path.display()
        );

        if matches!(middleware, Middleware::Json | Middleware::Toml) {
            log::warn!("Cannot syncback {middleware:?} at {inst_path}, skipping");
            continue;
        }

        let syncback = match middleware.syncback(&snapshot) {
            Ok(syncback) => syncback,
            Err(err) if middleware == Middleware::Dir => {
                let new_middleware = match env::var(DEBUG_MODEL_FORMAT_VAR) {
                    Ok(value) if value == "1" => Middleware::Rbxmx,
                    Ok(value) if value == "2" => Middleware::JsonModel,
                    _ => Middleware::Rbxm,
                };
                let file_name = snapshot
                    .path
                    .file_name()
                    .and_then(|s| s.to_str())
                    .context("Directory middleware should have a name in its path")?;
                let mut path = snapshot.path.clone();
                path.set_file_name(format!(
                    "{file_name}.{}",
                    extension_for_middleware(new_middleware)
                ));
                let new_snapshot = snapshot.with_new_path(path, snapshot.new, snapshot.old);
                // Record the fallback in stats instead of warning directly
                stats.record_rbxm_fallback(&inst_path, &err.to_string());
                let new_syncback_result = new_middleware
                    .syncback(&new_snapshot)
                    .with_context(|| format!("Failed to syncback {inst_path}"));
                if new_syncback_result.is_ok() && snapshot.old_inst().is_some() {
                    // We need to remove the old FS representation if we're
                    // reserializing it as an rbxm.
                    fs_snapshot.remove_dir(&snapshot.path);
                }
                new_syncback_result?
            }
            Err(err) => anyhow::bail!("Failed to syncback {inst_path} because {err}"),
        };

        if !syncback.removed_children.is_empty() {
            log::debug!(
                "removed children for {inst_path}: {}",
                syncback.removed_children.len()
            );
            'remove: for inst in &syncback.removed_children {
                let path = inst.metadata().instigating_source.as_ref().unwrap().path();
                let inst_path = snapshot.get_old_inst_path(inst.id());
                if !is_valid_path(&ignore_patterns, project_path, path) {
                    log::debug!(
                        "Skipping removing {} because its matches an ignore pattern",
                        path.display()
                    );
                    continue;
                }
                // Check ignoreTrees with glob pattern support
                if let Some(ref globs) = tree_globs {
                    for glob in globs {
                        if glob.is_match(&inst_path) {
                            log::debug!("Skipping removing {inst_path} because its path is blocked by ignoreTrees glob pattern");
                            continue 'remove;
                        }
                    }
                }
                if path.is_dir() {
                    fs_snapshot.remove_dir(path)
                } else {
                    fs_snapshot.remove_file(path)
                }
            }
        }

        // TODO provide replacement snapshots for e.g. two way sync

        fs_snapshot.merge_with_filter(syncback.fs_snapshot, |path| {
            is_valid_path(&ignore_patterns, project_path, path)
        });

        snapshots.extend(syncback.children);
    }

    // In clean mode, remove any existing paths that weren't overwritten by new structure.
    // This ensures no orphaned files remain from the old project layout.
    if !incremental && !existing_paths.is_empty() {
        log::debug!("Clean mode: checking for orphaned files to remove");

        // Helper to strip UNC prefix on Windows
        fn strip_unc_prefix(p: PathBuf) -> PathBuf {
            let path_str = p.to_string_lossy();
            if path_str.starts_with(r"\\?\") || path_str.starts_with("//?/") {
                PathBuf::from(&path_str[4..])
            } else {
                p
            }
        }

        // Helper to normalize a path by:
        // 1. Canonicalizing to resolve symlinks (important on macOS where /var -> /private/var)
        // 2. Stripping UNC prefix on Windows
        fn normalize_existing_path(p: &Path) -> PathBuf {
            // First try to canonicalize to resolve symlinks
            // This is important on macOS where /var is a symlink to /private/var
            let canonicalized = std::fs::canonicalize(p).unwrap_or_else(|_| p.to_path_buf());
            strip_unc_prefix(canonicalized)
        }

        // Canonicalize the project path base to resolve symlinks consistently
        // This is important on macOS where /var -> /private/var
        let canonical_project_path =
            std::fs::canonicalize(project_path).unwrap_or_else(|_| project_path.to_path_buf());
        let canonical_project_path = strip_unc_prefix(canonical_project_path);

        // Helper to canonicalize a path that may not exist yet.
        // Tries to canonicalize the full path, then falls back to canonicalizing
        // the nearest existing ancestor and appending the remaining components.
        fn canonicalize_maybe_nonexistent(p: &Path) -> PathBuf {
            // Try full canonicalization first (works if path exists)
            if let Ok(canonical) = std::fs::canonicalize(p) {
                return strip_unc_prefix(canonical);
            }

            // Path doesn't exist - find the nearest existing ancestor and canonicalize that,
            // then append the remaining path components.
            let mut existing_ancestor = p.to_path_buf();
            let mut suffix_components = Vec::new();

            while !existing_ancestor.as_os_str().is_empty() {
                if existing_ancestor.exists() {
                    break;
                }
                if let Some(file_name) = existing_ancestor.file_name() {
                    suffix_components.push(file_name.to_owned());
                }
                if !existing_ancestor.pop() {
                    break;
                }
            }

            // Canonicalize the existing ancestor
            let canonical_ancestor = std::fs::canonicalize(&existing_ancestor)
                .map(strip_unc_prefix)
                .unwrap_or(existing_ancestor);

            // Append the suffix components in reverse order
            let mut result = canonical_ancestor;
            for component in suffix_components.into_iter().rev() {
                result.push(component);
            }
            result
        }

        // Convert added_paths to absolute paths by joining with canonicalized project base path.
        // fs_snapshot paths are relative to the project folder.
        // We must also canonicalize the result to resolve any symlinks within the project
        // directory structure, ensuring consistent comparison with canonicalized existing paths.
        let added_paths: HashSet<PathBuf> = fs_snapshot
            .added_paths()
            .into_iter()
            .map(|p| canonicalize_maybe_nonexistent(&canonical_project_path.join(p)))
            .collect();

        // Get the project file path to exclude it from removal
        let project_file = normalize_existing_path(&project.file_location);

        // First pass: collect normalized paths to remove (deduplicated)
        let mut paths_to_remove: HashSet<PathBuf> = HashSet::new();
        for old_path in &existing_paths {
            let old_path_norm = normalize_existing_path(old_path);

            // Never remove the project file itself
            if old_path_norm == project_file {
                log::trace!("Skipping project file: {}", old_path.display());
                continue;
            }
            // Never remove any nested project files - they define project structure
            // and may be referenced via $path from other project files
            if let Some(file_name) = old_path_norm.file_name().and_then(|n| n.to_str()) {
                if file_name.ends_with(".project.json5") || file_name.ends_with(".project.json") {
                    log::trace!("Skipping nested project file: {}", old_path.display());
                    continue;
                }
            }
            // Skip if this exact path is being written to
            if added_paths.contains(&old_path_norm) {
                log::trace!("Skipping {} (exact path in added_paths)", old_path.display());
                continue;
            }
            // Skip if this path is an ancestor of something being written to
            // (e.g., a directory that will contain new files)
            // NOTE: We DO NOT skip files just because their parent directory is in added_paths.
            // Adding a directory does not mean all orphan files inside should be preserved.
            if old_path_norm.is_dir() {
                if added_paths.iter().any(|added| added.starts_with(&old_path_norm)) {
                    log::trace!("Skipping {} (ancestor of added path)", old_path.display());
                    continue;
                }
            }
            // Use normalized path to avoid duplicates from UNC vs regular paths
            log::trace!("Marking for removal: {}", old_path_norm.display());
            paths_to_remove.insert(old_path_norm);
        }

        // Second pass: only remove top-level orphaned paths.
        // Skip any path that is inside another path we're removing,
        // since remove_dir_all will handle the contents.
        let paths_vec: Vec<_> = paths_to_remove.iter().collect();
        for old_path in &paths_vec {
            // Check if this path is inside another path we're removing
            let is_nested = paths_vec
                .iter()
                .any(|other| *other != *old_path && old_path.starts_with(*other));

            if is_nested {
                continue;
            }

            // Convert absolute path to relative path for FsSnapshot.
            // FsSnapshot expects relative paths and joins them with base_path in write_to_vfs.
            let relative_path = old_path
                .strip_prefix(&canonical_project_path)
                .unwrap_or(old_path);

            log::debug!("Removing orphaned path: {}", relative_path.display());
            if old_path.is_dir() {
                fs_snapshot.remove_dir(relative_path);
            } else {
                fs_snapshot.remove_file(relative_path);
            }
        }
    }

    // Log stats summary if using internal stats (not provided externally)
    if external_stats.is_none() {
        stats.log_summary();
    }

    Ok(fs_snapshot)
}

pub struct SyncbackReturn<'sync> {
    pub fs_snapshot: FsSnapshot,
    pub children: Vec<SyncbackSnapshot<'sync>>,
    pub removed_children: Vec<InstanceWithMeta<'sync>>,
}

pub fn get_best_middleware(snapshot: &SyncbackSnapshot) -> Middleware {
    let old_middleware = snapshot
        .old_inst()
        .and_then(|inst| inst.metadata().middleware);
    let inst = snapshot.new_inst();

    let mut middleware;

    if let Some(override_middleware) = snapshot.middleware {
        return override_middleware;
    } else if let Some(old_middleware) = old_middleware {
        // Use old middleware, but upgrade to *Dir variant if new instance has children
        // This handles cases where the old file was a single file (e.g., Csv)
        // but the new instance has children (needs CsvDir)
        middleware = old_middleware;
    } else {
        // Specific classes that need special middleware, everything else defaults to JsonModel
        middleware = match inst.class.as_str() {
            "Folder" | "Configuration" | "Tool" | "ScreenGui" | "SurfaceGui" | "BillboardGui"
            | "AdGui" => Middleware::Dir,
            "StringValue" => Middleware::Text,
            "Script" => {
                // Check RunContext to determine which middleware to use
                // RunContext enum values: Legacy = 0, Server = 1, Client = 2, Plugin = 3
                match inst.properties.get(&ustr("RunContext")) {
                    Some(Variant::Enum(e)) => match e.to_u32() {
                        0 => Middleware::LegacyScript,
                        1 => Middleware::ServerScript,
                        2 => Middleware::ClientScript,
                        3 => Middleware::PluginScript,
                        _ => Middleware::LegacyScript, // Unknown RunContext, default to Legacy
                    },
                    _ => Middleware::LegacyScript, // No RunContext property, default to Legacy
                }
            }
            "LocalScript" => Middleware::LocalScript,
            "ModuleScript" => Middleware::ModuleScript,
            "LocalizationTable" => Middleware::Csv,
            // Default: use JsonModel for everything else (becomes Dir if has children)
            _ => Middleware::JsonModel,
        }
    }

    if !inst.children().is_empty() {
        middleware = match middleware {
            Middleware::ServerScript => Middleware::ServerScriptDir,
            Middleware::ClientScript => Middleware::ClientScriptDir,
            Middleware::ModuleScript => Middleware::ModuleScriptDir,
            Middleware::PluginScript => Middleware::PluginScriptDir,
            Middleware::LegacyScript => Middleware::LegacyScriptDir,
            Middleware::LocalScript => Middleware::LocalScriptDir,
            Middleware::Csv => Middleware::CsvDir,
            Middleware::JsonModel | Middleware::Text => Middleware::Dir,
            _ => middleware,
        }
    }

    middleware
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields, rename_all = "camelCase")]
pub struct SyncbackRules {
    /// A list of subtrees in a file that will be ignored by Syncback.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    ignore_trees: Vec<String>,
    /// A list of patterns to check against the path an Instance would serialize
    /// to. If a path matches one of these, the Instance won't be syncbacked.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    ignore_paths: Vec<String>,
    /// A map of classes to properties to ignore for that class when doing
    /// syncback.
    #[serde(default, skip_serializing_if = "IndexMap::is_empty")]
    ignore_properties: IndexMap<Ustr, Vec<Ustr>>,
    /// A list of class names to ignore entirely during syncback.
    /// Instances of these classes will not be added, removed, or synced.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    ignore_classes: Vec<String>,
    /// Whether or not the `CurrentCamera` of `Workspace` is included in the
    /// syncback or not. Defaults to `false`.
    #[serde(skip_serializing_if = "Option::is_none")]
    sync_current_camera: Option<bool>,
    /// Whether or not to sync properties that cannot be modified via scripts.
    /// Defaults to `true`.
    #[serde(skip_serializing_if = "Option::is_none")]
    sync_unscriptable: Option<bool>,
    /// Whether to skip serializing referent properties like `Model.PrimaryPart`
    /// during syncback. Defaults to `false`.
    #[serde(skip_serializing_if = "Option::is_none")]
    ignore_referents: Option<bool>,
    /// Whether the globs specified in `ignore_paths` should be modified to also
    /// match directories. Defaults to `true`.
    ///
    /// If this is `true`, it'll take ignore globs that end in `/**` and convert
    /// them to also handle the directory they're referring to. This is
    /// generally a better UX.
    #[serde(skip_serializing_if = "Option::is_none")]
    create_ignore_dir_paths: Option<bool>,
    /// Whether to encode Windows-invalid characters in file names during
    /// syncback. Characters like `<`, `>`, `:`, `"`, `/`, `\`, `|`, `?`, `*`
    /// will be encoded as `%LT%`, `%GT%`, `%COLON%`, etc.
    /// Defaults to `true`.
    #[serde(skip_serializing_if = "Option::is_none")]
    encode_windows_invalid_chars: Option<bool>,
    /// When enabled, only "visible" services will be synced back. This includes
    /// commonly used services like Workspace, ReplicatedStorage, ServerScriptService,
    /// etc., while ignoring internal/hidden services like Chat, HttpService, etc.
    /// Defaults to `true`.
    #[serde(skip_serializing_if = "Option::is_none")]
    ignore_hidden_services: Option<bool>,
    /// Whether to emit warnings when duplicate child names are encountered during
    /// syncback. Duplicate names (case-insensitive) cannot be reliably synced to
    /// the file system, so those children are skipped.
    /// Defaults to `false` (warnings are suppressed).
    #[serde(skip_serializing_if = "Option::is_none")]
    warn_duplicate_names: Option<bool>,
}

impl SyncbackRules {
    pub fn compile_globs(&self) -> anyhow::Result<Vec<Glob>> {
        let mut globs = Vec::with_capacity(self.ignore_paths.len());
        let dir_ignore_paths = self.create_ignore_dir_paths.unwrap_or(true);

        for pattern in &self.ignore_paths {
            let glob = Glob::new(pattern)
                .with_context(|| format!("the pattern '{pattern}' is not a valid glob"))?;
            globs.push(glob);

            if dir_ignore_paths {
                if let Some(dir_pattern) = pattern.strip_suffix("/**") {
                    if let Ok(glob) = Glob::new(dir_pattern) {
                        globs.push(glob)
                    }
                }
            }
        }

        Ok(globs)
    }

    /// Compiles the ignoreTrees patterns into glob matchers.
    /// Supports glob patterns like `**/Abc/Script` for flexible matching.
    pub fn compile_tree_globs(&self) -> anyhow::Result<Vec<Glob>> {
        let mut globs = Vec::with_capacity(self.ignore_trees.len());

        for pattern in &self.ignore_trees {
            let glob = Glob::new(pattern).with_context(|| {
                format!("the pattern '{pattern}' is not a valid ignoreTrees glob")
            })?;
            globs.push(glob);
        }

        Ok(globs)
    }

    /// Returns whether special characters (including periods) should be encoded
    /// in file names during syncback. Defaults to `true`.
    #[inline]
    pub fn encode_windows_invalid_chars(&self) -> bool {
        self.encode_windows_invalid_chars.unwrap_or(true)
    }

    /// Returns whether hidden/internal services should be ignored during
    /// syncback. When `true`, only visible services like Workspace,
    /// ReplicatedStorage, etc. will be synced. Defaults to `true`.
    #[inline]
    pub fn ignore_hidden_services(&self) -> bool {
        self.ignore_hidden_services.unwrap_or(true)
    }

    /// Returns whether to emit warnings when duplicate child names are
    /// encountered during syncback. Defaults to `false` (suppressed).
    #[inline]
    pub fn warn_duplicate_names(&self) -> bool {
        self.warn_duplicate_names.unwrap_or(false)
    }
}

fn is_valid_path(globs: &Option<Vec<Glob>>, base_path: &Path, path: &Path) -> bool {
    let git_glob = GIT_IGNORE_GLOB.get_or_init(|| Glob::new(".git/**").unwrap());
    let test_path = match path.strip_prefix(base_path) {
        Ok(suffix) => suffix,
        Err(_) => path,
    };
    if git_glob.is_match(test_path) {
        return false;
    }
    if let Some(ref ignore_paths) = globs {
        for glob in ignore_paths {
            if glob.is_match(test_path) {
                return false;
            }
        }
    }
    true
}

/// Returns a set of properties that should not be written with syncback if
/// one exists. This list is read directly from the Project and takes
/// inheritance into effect.
///
/// It **does not** handle properties that should not serialize for other
/// reasons, such as being defaults or being marked as not serializing in the
/// ReflectionDatabase.
fn get_property_filter(project: &Project, new_inst: &Instance) -> Option<UstrSet> {
    let filter = &project.syncback_rules.as_ref()?.ignore_properties;
    let mut set = UstrSet::default();

    let database = rbx_reflection_database::get().unwrap();
    let mut current_class_name = new_inst.class.as_str();

    loop {
        if let Some(list) = filter.get(&ustr(current_class_name)) {
            set.extend(list)
        }

        let class = database.classes.get(current_class_name)?;
        if let Some(super_class) = class.superclass.as_ref() {
            current_class_name = super_class;
        } else {
            break;
        }
    }

    Some(set)
}

/// Produces a list of descendants in the WeakDom such that all children come
/// before their parents.
fn descendants(dom: &WeakDom, root_ref: Ref) -> Vec<Ref> {
    let mut queue = VecDeque::new();
    let mut ordered = Vec::new();
    queue.push_front(root_ref);

    while let Some(referent) = queue.pop_front() {
        let inst = dom
            .get_by_ref(referent)
            .expect("Invariant: WeakDom had a Ref that wasn't inside it");
        ordered.push(referent);
        for child in inst.children() {
            queue.push_back(*child)
        }
    }

    ordered
}

/// Removes root children (services) that are not in the `VISIBLE_SERVICES` list.
/// This is used when `ignoreHiddenServices` is enabled to filter out internal
/// services like Chat, HttpService, etc.
///
/// This function only applies when the root is a DataModel (i.e., for place files).
/// For model files, the root children are regular instances, not services, so
/// filtering would incorrectly remove user content.
fn strip_hidden_services(dom: &mut WeakDom) {
    // Only apply service filtering when the root is a DataModel (place file)
    // For model files (rbxm), the root is typically not a DataModel and
    // children are regular instances, not services
    if dom.root().class != "DataModel" {
        log::trace!(
            "Skipping hidden services filter: root class is '{}', not 'DataModel'",
            dom.root().class
        );
        return;
    }

    let root_children = dom.root().children().to_vec();

    for child_ref in root_children {
        let child = dom
            .get_by_ref(child_ref)
            .expect("all children of the root should exist in the DOM");

        // Check if this service is in the visible services list
        if !VISIBLE_SERVICES.contains(&child.name.as_str()) {
            log::trace!(
                "Pruning hidden service {} of class {}",
                child.name,
                child.class
            );
            dom.destroy(child_ref);
        }
    }
}

/// Removes the children of `new`'s root that are not also children of `old`'s
/// root.
///
/// This does not care about duplicates, and only filters based on names and
/// class names.
fn strip_unknown_root_children(new: &mut WeakDom, old: &RojoTree) {
    let old_root = old.root();
    let old_root_children: HashMap<&str, InstanceWithMeta> = old_root
        .children()
        .iter()
        .map(|referent| {
            let inst = old
                .get_instance(*referent)
                .expect("all children of a DOM's root should exist");
            (inst.name(), inst)
        })
        .collect();

    let root_children = new.root().children().to_vec();

    for child_ref in root_children {
        let child = new
            .get_by_ref(child_ref)
            .expect("all children of the root should exist in the DOM");
        if let Some(old) = old_root_children.get(child.name.as_str()) {
            if old.class_name() == child.class {
                continue;
            }
        }
        log::trace!("Pruning root child {} of class {}", child.name, child.class);
        new.destroy(child_ref);
    }
}