Implement live syncback feature and enhance API for service data transfer

- Added a new `performSyncback` function in the `App` module to handle the synchronization of service data from the game to the server.
- Introduced a `SyncbackConfirm` UI component to confirm syncback actions from the user.
- Updated the API to include a new `/api/syncback` endpoint for handling syncback requests, validating protocol and server versions.
- Enhanced the `SyncbackSignal` mechanism to facilitate communication between the server and the syncback process.
- Added tests to ensure the functionality of the live syncback feature and its integration with the CLI.

These changes improve the overall synchronization capabilities of the application, allowing for real-time updates and better project management.
This commit is contained in:
Joe 2026-02-18 04:36:57 -06:00
parent 6f8e5a5cf2
commit 307ac81aec
17 changed files with 1411 additions and 97 deletions

View file

@ -470,6 +470,221 @@ fn get_port_number() -> usize {
port
}
/// Build a ServiceChunk by serializing child instances into rbxm format.
pub fn make_service_chunk(
class_name: &str,
children: Vec<rbx_dom_weak::InstanceBuilder>,
) -> librojo::web_api::ServiceChunk {
use rbx_dom_weak::WeakDom;
let mut dom = WeakDom::new(rbx_dom_weak::InstanceBuilder::new("DataModel"));
let root = dom.root_ref();
for child in children {
dom.insert(root, child);
}
let refs: Vec<Ref> = dom.root().children().to_vec();
let mut buf = Vec::new();
rbx_binary::to_writer(&mut buf, &dom, &refs).unwrap();
librojo::web_api::ServiceChunk {
class_name: class_name.to_string(),
data: buf,
}
}
/// Build a complete rbxl file from service chunks, mirroring the same data
/// that live syncback receives. Used for CLI parity testing.
pub fn make_rbxl_from_chunks(chunks: &[librojo::web_api::ServiceChunk]) -> Vec<u8> {
use rbx_dom_weak::WeakDom;
use std::collections::HashMap;
use std::io::Cursor;
let mut dom = WeakDom::new(rbx_dom_weak::InstanceBuilder::new("DataModel"));
let root_ref = dom.root_ref();
for chunk in chunks {
let chunk_dom = rbx_binary::from_reader(Cursor::new(&chunk.data))
.unwrap_or_else(|e| panic!("Failed to parse rbxm for {}: {e}", chunk.class_name));
let service_ref =
dom.insert(root_ref, rbx_dom_weak::InstanceBuilder::new(&chunk.class_name));
let mut ref_map: HashMap<Ref, Ref> = HashMap::new();
for &child_ref in chunk_dom.root().children() {
deep_clone_into_test(&chunk_dom, &mut dom, child_ref, service_ref, &mut ref_map);
}
let all_refs: Vec<Ref> = ref_map.values().copied().collect();
for inst_ref in all_refs {
let props_to_fix: Vec<(String, Ref)> = {
let inst = dom.get_by_ref(inst_ref).unwrap();
inst.properties
.iter()
.filter_map(|(key, value)| {
if let rbx_dom_weak::types::Variant::Ref(r) = value {
ref_map.get(r).map(|&mapped| (key.to_string(), mapped))
} else {
None
}
})
.collect()
};
if !props_to_fix.is_empty() {
let inst = dom.get_by_ref_mut(inst_ref).unwrap();
for (key, new_ref) in props_to_fix {
inst.properties
.insert(key.into(), rbx_dom_weak::types::Variant::Ref(new_ref));
}
}
}
}
let service_refs: Vec<Ref> = dom.root().children().to_vec();
let mut buf = Vec::new();
rbx_binary::to_writer(&mut buf, &dom, &service_refs).unwrap();
buf
}
fn deep_clone_into_test(
source: &rbx_dom_weak::WeakDom,
target: &mut rbx_dom_weak::WeakDom,
source_ref: Ref,
target_parent: Ref,
ref_map: &mut std::collections::HashMap<Ref, Ref>,
) {
let inst = source.get_by_ref(source_ref).unwrap();
let mut builder =
rbx_dom_weak::InstanceBuilder::new(inst.class.as_str()).with_name(inst.name.as_str());
for (key, value) in &inst.properties {
builder = builder.with_property(key.as_str(), value.clone());
}
let new_ref = target.insert(target_parent, builder);
ref_map.insert(source_ref, new_ref);
for &child_ref in inst.children() {
deep_clone_into_test(source, target, child_ref, new_ref, ref_map);
}
}
/// Run CLI syncback on a fresh copy of a fixture using the same data
/// that was sent to live syncback. Returns the temp dir (keep alive) and
/// project path.
pub fn run_cli_syncback_on_chunks(
fixture_name: &str,
chunks: &[librojo::web_api::ServiceChunk],
) -> (TempDir, PathBuf) {
use crate::rojo_test::io_util::{copy_recursive, ROJO_PATH, SERVE_TESTS_PATH};
let source_path = Path::new(SERVE_TESTS_PATH).join(fixture_name);
let dir = tempdir().expect("Couldn't create temp dir for CLI syncback");
let project_path = dir.path().join(fixture_name);
fs::create_dir(&project_path).expect("Couldn't create project subdirectory");
copy_recursive(&source_path, &project_path).expect("Couldn't copy fixture");
let rbxl_data = make_rbxl_from_chunks(chunks);
let rbxl_path = dir.path().join("input.rbxl");
fs::write(&rbxl_path, rbxl_data).expect("Failed to write rbxl");
let output = std::process::Command::new(ROJO_PATH)
.args([
"syncback",
project_path.to_str().unwrap(),
"--input",
rbxl_path.to_str().unwrap(),
])
.output()
.expect("Failed to run atlas syncback");
if !output.status.success() {
panic!(
"atlas syncback failed:\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
(dir, project_path)
}
impl TestServeSession {
/// Post to /api/syncback with raw bytes. Returns the full response
/// for status code inspection.
pub fn post_api_syncback_raw(&self, body: Vec<u8>) -> reqwest::blocking::Response {
let url = format!("http://localhost:{}/api/syncback", self.port);
let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(120))
.build()
.expect("Failed to build reqwest client");
client
.post(url)
.body(body)
.send()
.expect("Failed to send syncback request")
}
/// Post to /api/syncback with correct version/protocol info.
/// Panics if the response is not 200.
pub fn post_api_syncback(
&self,
place_id: Option<u64>,
services: Vec<librojo::web_api::ServiceChunk>,
) {
use serde::Serialize;
let request = serde_json::json!({
"protocolVersion": librojo::web_api::PROTOCOL_VERSION,
"serverVersion": env!("CARGO_PKG_VERSION"),
"placeId": place_id,
"services": services,
});
let mut body = Vec::new();
let mut serializer = rmp_serde::Serializer::new(&mut body)
.with_human_readable()
.with_struct_map();
request
.serialize(&mut serializer)
.expect("Failed to serialize syncback request");
let response = self.post_api_syncback_raw(body);
assert!(
response.status().is_success(),
"Syncback request failed with status {}: {}",
response.status(),
response.text().unwrap_or_default()
);
}
/// Wait for the server to come back online after a syncback-triggered
/// restart. Uses longer timeouts than initial startup since syncback
/// runs between teardown and restart.
pub fn wait_to_come_back_online(&self) -> ServerInfoResponse {
const BASE_DURATION_MS: f32 = 200.0;
const EXP_BACKOFF_FACTOR: f32 = 1.3;
const MAX_TRIES: u32 = 20;
for i in 1..=MAX_TRIES {
let info = match self.get_api_rojo() {
Ok(info) => info,
Err(_) => {
let retry_time_ms = BASE_DURATION_MS * (i as f32).powf(EXP_BACKOFF_FACTOR);
let retry_time = Duration::from_millis(retry_time_ms as u64);
thread::sleep(retry_time);
continue;
}
};
return info;
}
panic!(
"Server did not come back online after syncback ({} tries)",
MAX_TRIES
);
}
}
/// Extract the message cursor from a SocketPacket for use in subsequent
/// WebSocket subscriptions when chaining multi-step tests.
pub fn get_message_cursor(packet: &SocketPacket) -> u32 {

View file

@ -0,0 +1,536 @@
use std::{fs, thread, time::Duration};
use rbx_dom_weak::InstanceBuilder;
use crate::rojo_test::{
roundtrip_util::assert_dirs_equal,
serve_util::{make_service_chunk, run_cli_syncback_on_chunks, run_serve_test},
};
use librojo::web_api::{ServiceChunk, SocketPacketType, PROTOCOL_VERSION};
fn assert_live_matches_cli(
fixture_name: &str,
chunks: &[ServiceChunk],
place_id: Option<u64>,
) {
run_serve_test(fixture_name, |session, _| {
session.post_api_syncback(place_id, chunks.to_vec());
session.wait_to_come_back_online();
let (_cli_dir, cli_path) = run_cli_syncback_on_chunks(fixture_name, chunks);
assert_dirs_equal(session.path(), &cli_path);
});
}
// ── CLI parity tests ────────────────────────────────────────────
#[test]
fn parity_scripts() {
let chunks = vec![make_service_chunk(
"ReplicatedStorage",
vec![
InstanceBuilder::new("ModuleScript")
.with_name("Utils")
.with_property("Source", rbx_dom_weak::types::Variant::String("return {}".into())),
InstanceBuilder::new("Script")
.with_name("Main")
.with_property("Source", rbx_dom_weak::types::Variant::String("print('hi')".into()))
.with_property(
"RunContext",
rbx_dom_weak::types::Variant::Enum(rbx_dom_weak::types::Enum::from_u32(1)),
),
],
)];
assert_live_matches_cli("live_syncback", &chunks, None);
}
#[test]
fn parity_models_with_properties() {
let chunks = vec![make_service_chunk(
"Workspace",
vec![InstanceBuilder::new("Part")
.with_name("TestPart")
.with_property("Anchored", rbx_dom_weak::types::Variant::Bool(true))
.with_property(
"Size",
rbx_dom_weak::types::Variant::Vector3(rbx_dom_weak::types::Vector3::new(
4.0, 2.0, 6.0,
)),
)],
)];
assert_live_matches_cli("live_syncback", &chunks, None);
}
#[test]
fn parity_special_names() {
let chunks = vec![make_service_chunk(
"ReplicatedStorage",
vec![
InstanceBuilder::new("Folder").with_name("What?"),
InstanceBuilder::new("Folder").with_name("Key:Value"),
InstanceBuilder::new("Folder").with_name("A/B"),
],
)];
assert_live_matches_cli("live_syncback", &chunks, None);
}
#[test]
fn parity_duplicate_names() {
let chunks = vec![make_service_chunk(
"ReplicatedStorage",
vec![
InstanceBuilder::new("Folder").with_name("Data"),
InstanceBuilder::new("Folder").with_name("Data"),
InstanceBuilder::new("Folder").with_name("Data"),
],
)];
assert_live_matches_cli("live_syncback", &chunks, None);
}
#[test]
fn parity_deep_hierarchy() {
let chunks = vec![make_service_chunk(
"ReplicatedStorage",
vec![InstanceBuilder::new("Folder")
.with_name("A")
.with_children(vec![InstanceBuilder::new("Folder")
.with_name("B")
.with_children(vec![InstanceBuilder::new("Folder")
.with_name("C")
.with_children(vec![InstanceBuilder::new("Folder")
.with_name("D")
.with_children(vec![InstanceBuilder::new("ModuleScript")
.with_name("Leaf")
.with_property(
"Source",
rbx_dom_weak::types::Variant::String("return true".into()),
)])])])])],
)];
assert_live_matches_cli("live_syncback", &chunks, None);
}
#[test]
fn parity_mixed_file_types() {
let chunks = vec![make_service_chunk(
"ReplicatedStorage",
vec![
InstanceBuilder::new("StringValue")
.with_name("MyString")
.with_property("Value", rbx_dom_weak::types::Variant::String("hello world".into())),
InstanceBuilder::new("Folder").with_name("Container"),
InstanceBuilder::new("ModuleScript")
.with_name("DataModule")
.with_property(
"Source",
rbx_dom_weak::types::Variant::String("return {a=1}".into()),
),
],
)];
assert_live_matches_cli("live_syncback", &chunks, None);
}
#[test]
fn parity_multi_service() {
let chunks = vec![
make_service_chunk(
"ReplicatedStorage",
vec![InstanceBuilder::new("ModuleScript")
.with_name("SharedUtil")
.with_property(
"Source",
rbx_dom_weak::types::Variant::String("return {}".into()),
)],
),
make_service_chunk(
"ServerScriptService",
vec![InstanceBuilder::new("Script")
.with_name("ServerMain")
.with_property(
"Source",
rbx_dom_weak::types::Variant::String("print('server')".into()),
)
.with_property(
"RunContext",
rbx_dom_weak::types::Variant::Enum(rbx_dom_weak::types::Enum::from_u32(1)),
)],
),
make_service_chunk(
"Workspace",
vec![InstanceBuilder::new("Part")
.with_name("Floor")
.with_property("Anchored", rbx_dom_weak::types::Variant::Bool(true))],
),
];
assert_live_matches_cli("live_syncback", &chunks, None);
}
#[test]
fn parity_empty_services_mixed() {
let chunks = vec![
make_service_chunk(
"ReplicatedStorage",
vec![InstanceBuilder::new("Folder").with_name("Stuff")],
),
make_service_chunk("ServerScriptService", vec![]),
];
assert_live_matches_cli("live_syncback", &chunks, None);
}
// ── Validation / rejection tests ─────────────────────────────────
#[test]
fn rejects_bad_protocol() {
run_serve_test("live_syncback", |session, _| {
let initial_info = session.get_api_rojo().unwrap();
let request = serde_json::json!({
"protocolVersion": 9999,
"serverVersion": env!("CARGO_PKG_VERSION"),
"placeId": null,
"services": [],
});
let mut body = Vec::new();
let mut serializer = rmp_serde::Serializer::new(&mut body)
.with_human_readable()
.with_struct_map();
serde::Serialize::serialize(&request, &mut serializer).unwrap();
let response = session.post_api_syncback_raw(body);
assert!(
!response.status().is_success(),
"Expected rejection for bad protocol, got {}",
response.status()
);
let after_info = session.get_api_rojo().unwrap();
assert_eq!(
initial_info.session_id, after_info.session_id,
"Server should not have restarted"
);
});
}
#[test]
fn rejects_bad_version() {
run_serve_test("live_syncback", |session, _| {
let initial_info = session.get_api_rojo().unwrap();
let request = serde_json::json!({
"protocolVersion": PROTOCOL_VERSION,
"serverVersion": "0.0.0",
"placeId": null,
"services": [],
});
let mut body = Vec::new();
let mut serializer = rmp_serde::Serializer::new(&mut body)
.with_human_readable()
.with_struct_map();
serde::Serialize::serialize(&request, &mut serializer).unwrap();
let response = session.post_api_syncback_raw(body);
assert!(
!response.status().is_success(),
"Expected rejection for bad version, got {}",
response.status()
);
let after_info = session.get_api_rojo().unwrap();
assert_eq!(
initial_info.session_id, after_info.session_id,
"Server should not have restarted"
);
});
}
#[test]
fn rejects_blocked_place() {
run_serve_test("live_syncback_place_ids", |session, _| {
let initial_info = session.get_api_rojo().unwrap();
let request = serde_json::json!({
"protocolVersion": PROTOCOL_VERSION,
"serverVersion": env!("CARGO_PKG_VERSION"),
"placeId": 999,
"services": [],
});
let mut body = Vec::new();
let mut serializer = rmp_serde::Serializer::new(&mut body)
.with_human_readable()
.with_struct_map();
serde::Serialize::serialize(&request, &mut serializer).unwrap();
let response = session.post_api_syncback_raw(body);
assert_eq!(
response.status().as_u16(),
403,
"Expected 403 for blocked place"
);
let after_info = session.get_api_rojo().unwrap();
assert_eq!(
initial_info.session_id, after_info.session_id,
"Server should not have restarted"
);
});
}
#[test]
fn rejects_unlisted_place() {
run_serve_test("live_syncback_place_ids", |session, _| {
let initial_info = session.get_api_rojo().unwrap();
let request = serde_json::json!({
"protocolVersion": PROTOCOL_VERSION,
"serverVersion": env!("CARGO_PKG_VERSION"),
"placeId": 456,
"services": [],
});
let mut body = Vec::new();
let mut serializer = rmp_serde::Serializer::new(&mut body)
.with_human_readable()
.with_struct_map();
serde::Serialize::serialize(&request, &mut serializer).unwrap();
let response = session.post_api_syncback_raw(body);
assert_eq!(
response.status().as_u16(),
403,
"Expected 403 for unlisted place"
);
let after_info = session.get_api_rojo().unwrap();
assert_eq!(
initial_info.session_id, after_info.session_id,
"Server should not have restarted"
);
});
}
#[test]
fn allows_whitelisted_place() {
let chunks = vec![make_service_chunk(
"ReplicatedStorage",
vec![InstanceBuilder::new("Folder").with_name("Allowed")],
)];
run_serve_test("live_syncback_place_ids", |session, _| {
session.post_api_syncback(Some(123), chunks);
let new_info = session.wait_to_come_back_online();
assert!(
!new_info.project_name.is_empty(),
"Server should be back online with a project"
);
});
}
#[test]
fn allows_any_place_when_unrestricted() {
let chunks = vec![make_service_chunk(
"ReplicatedStorage",
vec![InstanceBuilder::new("Folder").with_name("Any")],
)];
run_serve_test("live_syncback", |session, _| {
session.post_api_syncback(Some(99999), chunks);
session.wait_to_come_back_online();
});
}
// ── Lifecycle / edge case tests ──────────────────────────────────
#[test]
fn server_comes_back_functional() {
let chunks = vec![make_service_chunk(
"ReplicatedStorage",
vec![InstanceBuilder::new("ModuleScript")
.with_name("TestModule")
.with_property(
"Source",
rbx_dom_weak::types::Variant::String("return 42".into()),
)],
)];
run_serve_test("live_syncback", |session, _| {
session.post_api_syncback(None, chunks);
let new_info = session.wait_to_come_back_online();
let read = session.get_api_read(new_info.root_instance_id).unwrap();
assert!(
!read.instances.is_empty(),
"Should be able to read instances from restarted server"
);
let module_path = session.path().join("src/shared/TestModule.luau");
let socket_packet = session.recv_socket_packet(SocketPacketType::Messages, 0, || {
fs::write(&module_path, "return 99").unwrap();
});
assert!(
socket_packet.is_ok(),
"WebSocket should receive patches after file modification"
);
});
}
#[test]
fn syncback_twice_different_data() {
let chunks_a = vec![make_service_chunk(
"ReplicatedStorage",
vec![InstanceBuilder::new("Folder").with_name("DataA")],
)];
let chunks_b = vec![make_service_chunk(
"ReplicatedStorage",
vec![InstanceBuilder::new("Folder").with_name("DataB")],
)];
run_serve_test("live_syncback", |session, _| {
session.post_api_syncback(None, chunks_a);
session.wait_to_come_back_online();
assert!(
session.path().join("src/shared/DataA").exists(),
"First syncback should create DataA"
);
session.post_api_syncback(None, chunks_b);
session.wait_to_come_back_online();
assert!(
!session.path().join("src/shared/DataA").exists(),
"DataA should be gone after second syncback (clean mode)"
);
assert!(
session.path().join("src/shared/DataB").exists(),
"Second syncback should create DataB"
);
});
}
#[test]
fn syncback_replaces_all_old_files() {
let chunks = vec![make_service_chunk(
"ReplicatedStorage",
vec![InstanceBuilder::new("ModuleScript")
.with_name("NewFile")
.with_property(
"Source",
rbx_dom_weak::types::Variant::String("return 'new'".into()),
)],
)];
run_serve_test("live_syncback", |session, _| {
assert!(
session.path().join("src/shared/OldModule.luau").exists(),
"OldModule should exist before syncback"
);
session.post_api_syncback(None, chunks);
session.wait_to_come_back_online();
assert!(
!session.path().join("src/shared/OldModule.luau").exists(),
"OldModule should be gone after clean syncback"
);
assert!(
session.path().join("src/shared/NewFile.luau").exists(),
"NewFile should exist after syncback"
);
});
}
// ── Round-trip identity test ─────────────────────────────────────
#[test]
fn roundtrip_build_syncback_rebuild() {
use crate::rojo_test::roundtrip_util::run_rojo_build;
use std::io::Cursor;
run_serve_test("live_syncback", |session, _| {
let (_build_dir, rbxl_path_a) = run_rojo_build(session.path(), "build_a.rbxl");
let rbxl_data_a = fs::read(&rbxl_path_a).unwrap();
let dom_a = rbx_binary::from_reader(Cursor::new(&rbxl_data_a)).unwrap();
let mut chunks = Vec::new();
for &service_ref in dom_a.root().children() {
let service = dom_a.get_by_ref(service_ref).unwrap();
let child_refs: Vec<rbx_dom_weak::types::Ref> = service.children().to_vec();
if child_refs.is_empty() {
continue;
}
let mut buf = Vec::new();
rbx_binary::to_writer(&mut buf, &dom_a, &child_refs).unwrap();
chunks.push(ServiceChunk {
class_name: service.class.to_string(),
data: buf,
});
}
session.post_api_syncback(None, chunks);
session.wait_to_come_back_online();
thread::sleep(Duration::from_millis(500));
let (_build_dir_b, rbxl_path_b) = run_rojo_build(session.path(), "build_b.rbxl");
let rbxl_data_b = fs::read(&rbxl_path_b).unwrap();
let dom_b = rbx_binary::from_reader(Cursor::new(&rbxl_data_b)).unwrap();
fn compare_trees(
dom_a: &rbx_dom_weak::WeakDom,
ref_a: rbx_dom_weak::types::Ref,
dom_b: &rbx_dom_weak::WeakDom,
ref_b: rbx_dom_weak::types::Ref,
path: &str,
) {
let inst_a = dom_a.get_by_ref(ref_a).unwrap();
let inst_b = dom_b.get_by_ref(ref_b).unwrap();
assert_eq!(
inst_a.name, inst_b.name,
"Name mismatch at {path}: {:?} vs {:?}",
inst_a.name, inst_b.name
);
assert_eq!(
inst_a.class, inst_b.class,
"Class mismatch at {path}: {:?} vs {:?}",
inst_a.class, inst_b.class
);
let mut children_a: Vec<_> = inst_a
.children()
.iter()
.map(|&r| {
let c = dom_a.get_by_ref(r).unwrap();
(c.name.clone(), c.class.clone(), r)
})
.collect();
children_a.sort_by(|a, b| (&a.0, &a.1).cmp(&(&b.0, &b.1)));
let mut children_b: Vec<_> = inst_b
.children()
.iter()
.map(|&r| {
let c = dom_b.get_by_ref(r).unwrap();
(c.name.clone(), c.class.clone(), r)
})
.collect();
children_b.sort_by(|a, b| (&a.0, &a.1).cmp(&(&b.0, &b.1)));
assert_eq!(
children_a.len(),
children_b.len(),
"Child count mismatch at {path}: {} vs {}",
children_a.len(),
children_b.len()
);
for (ca, cb) in children_a.iter().zip(children_b.iter()) {
let child_path = format!("{path}/{}", ca.0);
compare_trees(dom_a, ca.2, dom_b, cb.2, &child_path);
}
}
compare_trees(&dom_a, dom_a.root_ref(), &dom_b, dom_b.root_ref(), "root");
});
}

View file

@ -3,6 +3,7 @@ mod clean_mode;
mod clean_mode_stress;
mod connected_mode;
mod git_sync_defaults;
mod live_syncback;
mod matching_fixtures;
mod serve;
mod syncback;