Implement incremental syncback mode to preserve existing file structure

- Added an `incremental` flag to the `SyncbackCommand` to allow for preserving existing file structure and middleware formats during syncback operations.
- Enhanced the `syncback_loop` function to differentiate between clean mode and incremental mode, ensuring orphaned files are removed only in clean mode.
- Updated logging to reflect the mode of operation during syncback.
- Introduced new tests to validate the behavior of clean mode versus incremental mode, ensuring proper handling of orphaned files and preservation of existing structures.

tests/rojo_test/syncback_util.rs

@@ -11,7 +11,7 @@ const INPUT_FILE_PROJECT: &str = "input-project";
 const INPUT_FILE_PLACE: &str = "input.rbxl";
 const INPUT_FILE_MODEL: &str = "input.rbxm";
 
-/// Convenience method to run a `rojo syncback` test.
+/// Convenience method to run a `rojo syncback` test in clean mode (default).
 ///
 /// Test projects should be defined in the `syncback-tests` folder; their filename
 /// should be given as the first parameter.
@@ -19,6 +19,18 @@ const INPUT_FILE_MODEL: &str = "input.rbxm";
 /// The passed in callback is where the actual test body should go. Setup and
 /// cleanup happens automatically.
 pub fn run_syncback_test(name: &str, callback: impl FnOnce(&Path)) {
+    run_syncback_test_impl(name, false, callback)
+}
+
+/// Convenience method to run a `rojo syncback` test in incremental mode.
+///
+/// Use this for tests that depend on preserving existing file structure and
+/// middleware formats.
+pub fn run_syncback_test_incremental(name: &str, callback: impl FnOnce(&Path)) {
+    run_syncback_test_impl(name, true, callback)
+}
+
+fn run_syncback_test_impl(name: &str, incremental: bool, callback: impl FnOnce(&Path)) {
     let _ = env_logger::try_init();
 
     // let working_dir = get_working_dir_path();
@@ -55,21 +67,27 @@ pub fn run_syncback_test(name: &str, callback: impl FnOnce(&Path)) {
             .expect("Couldn't copy project to temporary directory");
     };
 
+    let mut args = vec![
+        "--color",
+        "never",
+        "syncback",
+        project_path.to_str().unwrap(),
+        "--input",
+        input_file.to_str().unwrap(),
+        "--non-interactive",
+        "--list",
+    ];
+
+    if incremental {
+        args.push("--incremental");
+    }
+
     let output = Command::new(ROJO_PATH)
         // I don't really understand why setting the working directory breaks this, but it does.
         // It's a bit concerning but I'm more interested in writing tests than debugging it right now.
         // TODO: Figure out why and fix it.
         // .current_dir(working_dir)
-        .args([
-            "--color",
-            "never",
-            "syncback",
-            project_path.to_str().unwrap(),
-            "--input",
-            input_file.to_str().unwrap(),
-            "--non-interactive",
-            "--list",
-        ])
+        .args(args)
         .output()
         .expect("Couldn't spawn syncback process");
 

tests/tests/clean_mode.rs

@@ -0,0 +1,322 @@
+//! Tests for clean mode (default) vs incremental mode behavior.
+//!
+//! These tests verify that:
+//! 1. Clean mode removes orphaned files
+//! 2. Clean mode doesn't remove files that are being written to
+//! 3. Clean mode doesn't remove parent directories of files being written
+//! 4. Incremental mode preserves existing file structure
+
+use std::path::Path;
+use std::process::Command;
+
+use tempfile::tempdir;
+
+use crate::rojo_test::io_util::{copy_recursive, ROJO_PATH};
+
+const SYNCBACK_TESTS_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/rojo-test/syncback-tests");
+
+/// Helper to run syncback and return success status
+fn run_syncback(project_path: &Path, input_path: &Path, incremental: bool) -> bool {
+    let mut args = vec![
+        "--color",
+        "never",
+        "syncback",
+        project_path.to_str().unwrap(),
+        "--input",
+        input_path.to_str().unwrap(),
+        "--non-interactive",
+    ];
+
+    if incremental {
+        args.push("--incremental");
+    }
+
+    let output = Command::new(ROJO_PATH)
+        .args(args)
+        .output()
+        .expect("Couldn't spawn syncback process");
+
+    if !output.status.success() {
+        eprintln!("Syncback failed!");
+        eprintln!("stdout: {}", String::from_utf8_lossy(&output.stdout));
+        eprintln!("stderr: {}", String::from_utf8_lossy(&output.stderr));
+    }
+
+    output.status.success()
+}
+
+/// Test that clean mode removes orphaned files that don't exist in the input
+#[test]
+fn clean_mode_removes_orphaned_files() {
+    let _ = env_logger::try_init();
+
+    // Use the sync_rules test case and add an orphaned file
+    let source_path = Path::new(SYNCBACK_TESTS_PATH)
+        .join("sync_rules")
+        .join("input-project");
+    let input_file = Path::new(SYNCBACK_TESTS_PATH)
+        .join("sync_rules")
+        .join("input.rbxm");
+
+    let test_dir = tempdir().expect("Couldn't create temporary directory");
+    let project_path = test_dir.path().join("test_project");
+
+    fs_err::create_dir(&project_path).expect("Couldn't create project directory");
+    copy_recursive(&source_path, &project_path).expect("Couldn't copy project");
+
+    // Create an orphaned file that doesn't match anything in the input
+    let orphaned_file = project_path.join("src").join("OrphanedFile.luau");
+    fs_err::write(&orphaned_file, "-- This is orphaned").expect("Couldn't create orphaned file");
+
+    assert!(
+        orphaned_file.exists(),
+        "Orphaned file should exist before syncback"
+    );
+
+    // Run syncback in clean mode (default)
+    assert!(
+        run_syncback(&project_path, &input_file, false),
+        "Syncback should succeed"
+    );
+
+    // The orphaned file should be removed in clean mode
+    // (the input doesn't have "OrphanedFile" script)
+    assert!(
+        !orphaned_file.exists(),
+        "Orphaned file should be removed in clean mode"
+    );
+
+    // The expected files should exist
+    let module_file = project_path.join("src").join("module.luau");
+    assert!(
+        module_file.exists(),
+        "module.luau should be created by syncback"
+    );
+}
+
+/// Test that clean mode doesn't remove files that are being written to
+#[test]
+fn clean_mode_preserves_written_files() {
+    let _ = env_logger::try_init();
+
+    // Use the child_but_not test case
+    let source_path = Path::new(SYNCBACK_TESTS_PATH)
+        .join("child_but_not")
+        .join("input-project");
+    let input_file = Path::new(SYNCBACK_TESTS_PATH)
+        .join("child_but_not")
+        .join("input.rbxl");
+
+    let test_dir = tempdir().expect("Couldn't create temporary directory");
+    let project_path = test_dir.path().join("test_project");
+
+    fs_err::create_dir(&project_path).expect("Couldn't create project directory");
+    copy_recursive(&source_path, &project_path).expect("Couldn't copy project");
+
+    // Run syncback in clean mode
+    assert!(
+        run_syncback(&project_path, &input_file, false),
+        "Syncback should succeed"
+    );
+
+    // The directories being written to should exist
+    let only_one_copy = project_path.join("OnlyOneCopy");
+    let replicated_storage = project_path.join("ReplicatedStorage");
+
+    assert!(
+        only_one_copy.exists(),
+        "OnlyOneCopy directory should exist (it's being written to)"
+    );
+    assert!(
+        replicated_storage.exists(),
+        "ReplicatedStorage directory should exist (it's being written to)"
+    );
+
+    // The files inside should exist
+    assert!(
+        only_one_copy.join("child_of_one.luau").exists(),
+        "child_of_one.luau should exist"
+    );
+    assert!(
+        replicated_storage
+            .join("child_replicated_storage.luau")
+            .exists(),
+        "child_replicated_storage.luau should exist"
+    );
+}
+
+/// Test that incremental mode preserves existing file structure
+#[test]
+fn incremental_mode_preserves_structure() {
+    let _ = env_logger::try_init();
+
+    // Use the sync_rules test case - it has custom extensions
+    let source_path = Path::new(SYNCBACK_TESTS_PATH)
+        .join("sync_rules")
+        .join("input-project");
+    let input_file = Path::new(SYNCBACK_TESTS_PATH)
+        .join("sync_rules")
+        .join("input.rbxm");
+
+    let test_dir = tempdir().expect("Couldn't create temporary directory");
+    let project_path = test_dir.path().join("test_project");
+
+    fs_err::create_dir(&project_path).expect("Couldn't create project directory");
+    copy_recursive(&source_path, &project_path).expect("Couldn't copy project");
+
+    // Verify the custom-extension files exist before syncback
+    let modulescript = project_path.join("src").join("module.modulescript");
+    let text = project_path.join("src").join("text.text");
+    assert!(
+        modulescript.exists(),
+        "module.modulescript should exist before syncback"
+    );
+    assert!(text.exists(), "text.text should exist before syncback");
+
+    // Run syncback in incremental mode
+    assert!(
+        run_syncback(&project_path, &input_file, true),
+        "Syncback should succeed"
+    );
+
+    // In incremental mode, the custom extensions should be preserved
+    assert!(
+        modulescript.exists(),
+        "module.modulescript should be preserved in incremental mode"
+    );
+    assert!(
+        text.exists(),
+        "text.text should be preserved in incremental mode"
+    );
+}
+
+/// Test that clean mode uses fresh file extensions (not preserving old ones)
+#[test]
+fn clean_mode_uses_fresh_extensions() {
+    let _ = env_logger::try_init();
+
+    // Use the sync_rules test case - it has custom extensions
+    let source_path = Path::new(SYNCBACK_TESTS_PATH)
+        .join("sync_rules")
+        .join("input-project");
+    let input_file = Path::new(SYNCBACK_TESTS_PATH)
+        .join("sync_rules")
+        .join("input.rbxm");
+
+    let test_dir = tempdir().expect("Couldn't create temporary directory");
+    let project_path = test_dir.path().join("test_project");
+
+    fs_err::create_dir(&project_path).expect("Couldn't create project directory");
+    copy_recursive(&source_path, &project_path).expect("Couldn't copy project");
+
+    // Run syncback in clean mode (default)
+    assert!(
+        run_syncback(&project_path, &input_file, false),
+        "Syncback should succeed"
+    );
+
+    // In clean mode, the files should use standard extensions
+    let modulescript_old = project_path.join("src").join("module.modulescript");
+    let text_old = project_path.join("src").join("text.text");
+    let module_new = project_path.join("src").join("module.luau");
+    let text_new = project_path.join("src").join("text.txt");
+
+    // Old extensions should NOT exist
+    assert!(
+        !modulescript_old.exists(),
+        "module.modulescript should NOT exist in clean mode"
+    );
+    assert!(
+        !text_old.exists(),
+        "text.text should NOT exist in clean mode"
+    );
+
+    // New standard extensions SHOULD exist
+    assert!(
+        module_new.exists(),
+        "module.luau should exist in clean mode"
+    );
+    assert!(text_new.exists(), "text.txt should exist in clean mode");
+}
+
+/// Test that clean mode doesn't remove the project file
+#[test]
+fn clean_mode_preserves_project_file() {
+    let _ = env_logger::try_init();
+
+    let source_path = Path::new(SYNCBACK_TESTS_PATH)
+        .join("ignore_trees_adding")
+        .join("input-project");
+    let input_file = Path::new(SYNCBACK_TESTS_PATH)
+        .join("ignore_trees_adding")
+        .join("input.rbxl");
+
+    let test_dir = tempdir().expect("Couldn't create temporary directory");
+    let project_path = test_dir.path().join("test_project");
+
+    fs_err::create_dir(&project_path).expect("Couldn't create project directory");
+    copy_recursive(&source_path, &project_path).expect("Couldn't copy project");
+
+    let project_file = project_path.join("default.project.json5");
+    assert!(
+        project_file.exists(),
+        "Project file should exist before syncback"
+    );
+
+    // Run syncback in clean mode
+    assert!(
+        run_syncback(&project_path, &input_file, false),
+        "Syncback should succeed"
+    );
+
+    // Project file should still exist
+    assert!(
+        project_file.exists(),
+        "Project file should be preserved in clean mode"
+    );
+}
+
+/// Test that running clean mode twice produces identical results
+#[test]
+fn clean_mode_is_idempotent() {
+    let _ = env_logger::try_init();
+
+    let source_path = Path::new(SYNCBACK_TESTS_PATH)
+        .join("child_but_not")
+        .join("input-project");
+    let input_file = Path::new(SYNCBACK_TESTS_PATH)
+        .join("child_but_not")
+        .join("input.rbxl");
+
+    let test_dir = tempdir().expect("Couldn't create temporary directory");
+    let project_path = test_dir.path().join("test_project");
+
+    fs_err::create_dir(&project_path).expect("Couldn't create project directory");
+    copy_recursive(&source_path, &project_path).expect("Couldn't copy project");
+
+    // Run syncback twice in clean mode
+    assert!(
+        run_syncback(&project_path, &input_file, false),
+        "First syncback should succeed"
+    );
+
+    // Capture state after first run
+    let file1 = project_path.join("OnlyOneCopy").join("child_of_one.luau");
+    let content1 = fs_err::read_to_string(&file1).unwrap();
+
+    assert!(
+        run_syncback(&project_path, &input_file, false),
+        "Second syncback should succeed"
+    );
+
+    // File should still exist with same content
+    assert!(
+        file1.exists(),
+        "File should still exist after second syncback"
+    );
+    let content2 = fs_err::read_to_string(&file1).unwrap();
+    assert_eq!(
+        content1, content2,
+        "File content should be identical after second syncback"
+    );
+}

tests/tests/mod.rs

@@ -1,3 +1,4 @@
 mod build;
+mod clean_mode;
 mod serve;
 mod syncback;

tests/tests/syncback.rs

@@ -2,7 +2,9 @@ use std::ffi::OsStr;
 
 use insta::assert_snapshot;
 
-use crate::rojo_test::syncback_util::{run_syncback_test, snapshot_rbxm};
+use crate::rojo_test::syncback_util::{
+    run_syncback_test, run_syncback_test_incremental, snapshot_rbxm,
+};
 
 macro_rules! syncback_tests {
     ($($test_name:ident => $list:expr$(,)?),*) => {$(
@@ -28,27 +30,47 @@ macro_rules! syncback_tests {
     )*};
 }
 
+/// Tests that need to run in incremental mode (preserving existing structure)
+macro_rules! syncback_tests_incremental {
+    ($($test_name:ident => $list:expr$(,)?),*) => {$(
+        #[test]
+        fn $test_name() {
+            run_syncback_test_incremental(stringify!($test_name), |path| {
+                for name in $list {
+                    let snapshot_name = format!(concat!(stringify!($test_name), "-{}"), name);
+                    let new = path.join::<&str>(name);
+                    if !new.exists() {
+                        panic!("the path stub '{}' does not exist after syncback runs. consider double checking for typos.", name);
+                    }
+                    if let Some("rbxm") = new.extension().and_then(OsStr::to_str) {
+                        let content = fs_err::read(new).unwrap();
+                        snapshot_rbxm(&snapshot_name, content, name);
+                    } else {
+                        let content = fs_err::read_to_string(new).unwrap();
+                        assert_snapshot!(snapshot_name, content, name);
+                    }
+                }
+            });
+        }
+    )*};
+}
+
+// Tests that run in clean mode (default)
 syncback_tests! {
     // Ensures that there's only one copy written to disk if navigating a
     // project file might yield two copies
     child_but_not => ["OnlyOneCopy/child_of_one.luau", "ReplicatedStorage/child_replicated_storage.luau"],
-    // Ensures that syncback works with CSVs
-    csv => ["src/csv_init/init.csv", "src/csv.csv"],
     // Ensures that if a RojoId is duplicated somewhere in the project, it's
     // rewritten rather than synced back as a conflict
     duplicate_rojo_id => ["container.model.json5"],
     // Ensures that the `ignorePaths` setting works for additions (new files get .json5)
     ignore_paths_adding => ["src/int_value.model.json5", "src/subfolder/string_value.txt"],
-    // Ensures that the `ignorePaths` setting works for `init` files
-    ignore_paths_init => ["src/non-init.luau", "src/init-file/init.luau"],
     // Ensures that the `ignorePaths` setting works for removals (new file gets .json5)
     ignore_paths_removing => ["src/Message.model.json5"],
     // Ensures that `ignoreTrees` works for additions
     ignore_trees_adding => [],
     // Ensures that `ignoreTrees` works for removals
     ignore_trees_removing => [],
-    // Ensures that all of the JSON middlewares are handled as expected
-    json_middlewares => ["src/dir_with_meta/init.meta.json5", "src/model_json.model.json5", "src/project_json.project.json5"],
     // Ensures projects that refer to other projects work as expected
     nested_projects => ["nested.project.json5", "string_value.txt"],
     // Ensures files that are ignored by nested projects are picked up if
@@ -62,28 +84,37 @@ syncback_tests! {
     project_reserialize => ["attribute_mismatch.luau", "property_mismatch.project.json5"],
     // Confirms that Instances that cannot serialize as directories serialize as rbxms
     rbxm_fallback => [],
-    // Ensures that ref properties are linked properly on the file system
-    ref_properties => ["src/pointer.model.json5", "src/target.model.json5"],
     // Ensures that ref properties are linked when no attributes are manually
     // set in the DataModel (new files get .json5)
     ref_properties_blank => ["src/pointer.model.json5", "src/target.txt"],
-    // Ensures that if there is a conflict in RojoRefs, one of them is rewritten
-    ref_properties_conflict => ["src/Pointer_2.model.json5", "src/Target_2.model.json5"],
     // Ensures that having multiple pointers that are aimed at the same target doesn't trigger ref rewrites.
     ref_properties_duplicate => [],
     // Ensures that ref properties that point to nothing after the prune both
     // do not leave any trace of themselves (new files get .json5)
     ref_properties_pruned => ["src/Pointer1.model.json5", "src/Pointer2.model.json5", "src/Pointer3.model.json5"],
-    // Ensures that the old middleware is respected during syncback
-    respect_old_middleware => ["default.project.json5", "src/model_json.model.json5", "src/rbxm.rbxm", "src/rbxmx.rbxmx"],
     // Ensures that the `$schema` field roundtrips with syncback
     schema_roundtrip => ["default.project.json5", "src/model.model.json5", "src/init/init.meta.json5", "src/adjacent.meta.json5"],
     // Ensures that StringValues inside project files are written to the
     // project file, but only if they don't have `$path` set
     string_value_project => ["default.project.json5"],
-    // Ensures that sync rules are respected. This is really just a test to
-    // ensure it uses the old path when possible, but we want the coverage.
-    sync_rules => ["src/module.modulescript", "src/text.text"],
     // Ensures that the `syncUnscriptable` setting works
     unscriptable_properties => ["default.project.json5"],
 }
+
+// Tests that run in incremental mode (preserving existing structure)
+syncback_tests_incremental! {
+    // Ensures that syncback works with CSVs (preserves existing CSV structure)
+    csv => ["src/csv_init/init.csv", "src/csv.csv"],
+    // Ensures that the `ignorePaths` setting works for `init` files
+    ignore_paths_init => ["src/non-init.luau", "src/init-file/init.luau"],
+    // Ensures that all of the JSON middlewares are handled as expected
+    json_middlewares => ["src/dir_with_meta/init.meta.json5", "src/model_json.model.json5", "src/project_json.project.json5"],
+    // Ensures that ref properties are linked properly on the file system
+    ref_properties => ["src/pointer.model.json5", "src/target.model.json5"],
+    // Ensures that if there is a conflict in RojoRefs, one of them is rewritten
+    ref_properties_conflict => ["src/Pointer_2.model.json5", "src/Target_2.model.json5"],
+    // Ensures that the old middleware is respected during syncback (incremental mode only)
+    respect_old_middleware => ["default.project.json5", "src/model_json.model.json5", "src/rbxm.rbxm", "src/rbxmx.rbxmx"],
+    // Ensures that sync rules are respected (incremental mode only - uses old paths when possible)
+    sync_rules => ["src/module.modulescript", "src/text.text"],
+}