diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 581be871..f0a6e9d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,13 @@ jobs: with: name: Rojo.rbxm path: Rojo.rbxm + + - name: Upload Plugin to Roblox + env: + RBX_API_KEY: ${{ secrets.PLUGIN_UPLOAD_TOKEN }} + RBX_UNIVERSE_ID: ${{ vars.PLUGIN_CI_PLACE_ID }} + RBX_PLACE_ID: ${{ vars.PLUGIN_CI_UNIVERSE_ID }} + run: lune run upload-plugin Rojo.rbxm build: needs: ["create-release"] diff --git a/.gitmodules b/.gitmodules index 46789ed1..59bf5d32 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,9 @@ [submodule "plugin/Packages/Highlighter"] path = plugin/Packages/Highlighter url = https://github.com/boatbomber/highlighter.git +[submodule "plugin/Packages/msgpack-luau"] + path = plugin/Packages/msgpack-luau + url = https://github.com/cipharius/msgpack-luau/ +[submodule ".lune/opencloud-execute"] + path = .lune/opencloud-execute + url = https://github.com/Dekkonot/opencloud-luau-execute-lune.git diff --git a/.lune/.config.luau b/.lune/.config.luau new file mode 100644 index 00000000..43665fdb --- /dev/null +++ b/.lune/.config.luau @@ -0,0 +1,8 @@ +return { + luau = { + languagemode = "strict", + aliases = { + lune = "~/.lune/.typedefs/0.10.4/", + }, + }, +} diff --git a/.lune/opencloud-execute b/.lune/opencloud-execute new file mode 160000 index 00000000..8ae86dd3 --- /dev/null +++ b/.lune/opencloud-execute @@ -0,0 +1 @@ +Subproject commit 8ae86dd3ad35eb66b0a4edda7e8771a5986cd0c6 diff --git a/.lune/scripts/plugin-upload.luau b/.lune/scripts/plugin-upload.luau new file mode 100644 index 00000000..c5708796 --- /dev/null +++ b/.lune/scripts/plugin-upload.luau @@ -0,0 +1,51 @@ +local args: any = ... +assert(args, "no arguments passed to script") + +local input: buffer = args.BinaryInput + +local AssetService = game:GetService("AssetService") +local SerializationService = game:GetService("SerializationService") +local EncodingService = game:GetService("EncodingService") + +local input_hash: buffer = EncodingService:ComputeBufferHash(input, Enum.HashAlgorithm.Sha256) +local hex_hash: { string } = table.create(buffer.len(input_hash)) +for i = 0, buffer.len(input_hash) - 1 do + table.insert(hex_hash, string.format("%02x", buffer.readu8(input_hash, i))) +end + +print(`Deserializing plugin file (size: {buffer.len(input)} bytes, hash: {table.concat(hex_hash, "")})`) +local plugin = SerializationService:DeserializeInstancesAsync(input)[1] + +local UploadDetails = require(plugin.UploadDetails) :: any +local PLUGIN_ID = UploadDetails.assetId +local PLUGIN_NAME = UploadDetails.name +local PLUGIN_DESCRIPTION = UploadDetails.description +local PLUGIN_CREATOR_ID = UploadDetails.creatorId +local PLUGIN_CREATOR_TYPE = UploadDetails.creatorType + +assert(typeof(PLUGIN_ID) == "number", "UploadDetails did not contain a number field 'assetId'") +assert(typeof(PLUGIN_NAME) == "string", "UploadDetails did not contain a string field 'name'") +assert(typeof(PLUGIN_DESCRIPTION) == "string", "UploadDetails did not contain a string field 'description'") +assert(typeof(PLUGIN_CREATOR_ID) == "number", "UploadDetails did not contain a number field 'creatorId'") +assert(typeof(PLUGIN_CREATOR_TYPE) == "string", "UploadDetails did not contain a string field 'creatorType'") +assert( + Enum.AssetCreatorType:FromName(PLUGIN_CREATOR_TYPE) ~= nil, + "UploadDetails field 'creatorType' was not a valid member of Enum.AssetCreatorType" +) + +print(`Uploading to {PLUGIN_ID}`) +print(`Plugin Name: {PLUGIN_NAME}`) +print(`Plugin Description: {PLUGIN_DESCRIPTION}`) + +local result, version_or_err = AssetService:CreateAssetVersionAsync(plugin, Enum.AssetType.Plugin, PLUGIN_ID, { + ["Name"] = PLUGIN_NAME, + ["Description"] = PLUGIN_DESCRIPTION, + ["CreatorId"] = PLUGIN_CREATOR_ID, + ["CreatorType"] = Enum.AssetCreatorType:FromName(PLUGIN_CREATOR_TYPE), +}) + +if result ~= Enum.CreateAssetResult.Success then + error(`Plugin failed to upload because: {result.Name} - {version_or_err}`) +end + +print(`Plugin uploaded successfully. New version is {version_or_err}.`) diff --git a/.lune/upload-plugin.luau b/.lune/upload-plugin.luau new file mode 100644 index 00000000..e71acf6c --- /dev/null +++ b/.lune/upload-plugin.luau @@ -0,0 +1,78 @@ +local fs = require("@lune/fs") +local process = require("@lune/process") +local stdio = require("@lune/stdio") + +local luau_execute = require("./opencloud-execute") + +local UNIVERSE_ID = process.env["RBX_UNIVERSE_ID"] +local PLACE_ID = process.env["RBX_PLACE_ID"] + +local version_string = fs.readFile("plugin/Version.txt") +local versions = { string.match(version_string, "^v?(%d+)%.(%d+)%.(%d+)(.*)$") } +if versions[4] ~= "" then + print("This release is a pre-release. Skipping uploading plugin.") + process.exit(0) +end + +local plugin_path = process.args[1] +assert( + typeof(plugin_path) == "string", + "no plugin path provided, expected usage is `lune run upload-plugin [PATH TO RBXM]`." +) + +-- For local testing +if process.env["CI"] ~= "true" then + local rojo = process.exec("rojo", { "build", "plugin.project.json", "--output", plugin_path }) + if not rojo.ok then + stdio.ewrite("plugin upload failed because: could not build plugin.rbxm\n\n") + stdio.ewrite(rojo.stderr) + stdio.ewrite("\n") + process.exit(1) + end +else + assert(fs.isFile(plugin_path), `Plugin file did not exist at {plugin_path}`) +end +local plugin_content = fs.readFile(plugin_path) + +local engine_script = fs.readFile(".lune/scripts/plugin-upload.luau") + +print("Creating task to upload plugin") +local task = luau_execute.create_task_latest(UNIVERSE_ID, PLACE_ID, engine_script, 300, false, plugin_content) + +print("Waiting for task to finish") +local success = luau_execute.await_finish(task) +if not success then + local error = luau_execute.get_error(task) + assert(error, "could not fetch error from task") + stdio.ewrite("plugin upload failed because: task did not finish successfully\n\n") + stdio.ewrite(error.code) + stdio.ewrite("\n") + stdio.ewrite(error.message) + stdio.ewrite("\n") + process.exit(1) +end + +print("Output from task:\n") +for _, log in luau_execute.get_structured_logs(task) do + if log.messageType == "ERROR" then + stdio.write(stdio.color("red")) + stdio.write(log.message) + stdio.write("\n") + stdio.write(stdio.color("reset")) + elseif log.messageType == "INFO" then + stdio.write(stdio.color("cyan")) + stdio.write(log.message) + stdio.write("\n") + stdio.write(stdio.color("reset")) + elseif log.messageType == "WARNING" then + stdio.write(stdio.color("yellow")) + stdio.write(log.message) + stdio.write("\n") + stdio.write(stdio.color("reset")) + else + stdio.write(stdio.color("reset")) + stdio.write(log.message) + stdio.write("\n") + stdio.write(stdio.color("reset")) + end +end diff --git a/CHANGELOG.md b/CHANGELOG.md index edfa85df..92e955b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,18 +32,24 @@ Making a new release? Simply add the new header with the version and date undern ## Unreleased * Added headless API for Studio companion plugins. ([#639]) +* `inf` and `nan` values in properties are now synced ([#1176]) * Fixed a bug caused by having reference properties (such as `ObjectValue.Value`) that point to an Instance not included in syncback. ([#1179]) * Fixed instance replacement fallback failing when too many instances needed to be replaced. ([#1192]) +* Added actors and bindable/remote event/function variants to be synced back as JSON files. ([#1199]) * Fixed a bug where MacOS paths weren't being handled correctly. ([#1201]) * Fixed a bug where the notification timeout thread would fail to cancel on unmount ([#1211]) * Added a "Forget" option to the sync reminder notification to avoid being reminded for that place in the future ([#1215]) +* Improves relative path calculation for sourcemap generation to avoid issues with Windows UNC paths. ([#1217]) [#639]: https://github.com/rojo-rbx/rojo/pull/639 +[#1176]: https://github.com/rojo-rbx/rojo/pull/1176 [#1179]: https://github.com/rojo-rbx/rojo/pull/1179 [#1192]: https://github.com/rojo-rbx/rojo/pull/1192 +[#1199]: https://github.com/rojo-rbx/rojo/pull/1199 [#1201]: https://github.com/rojo-rbx/rojo/pull/1201 [#1211]: https://github.com/rojo-rbx/rojo/pull/1211 [#1215]: https://github.com/rojo-rbx/rojo/pull/1215 +[#1217]: https://github.com/rojo-rbx/rojo/pull/1217 ## [7.7.0-rc.1] (November 27th, 2025) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c050a223..0063e448 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,6 +14,7 @@ Code contributions are welcome for features and bugs that have been reported in You'll want these tools to work on Rojo: * Latest stable Rust compiler + * Rustfmt and Clippy are used for code formatting and linting. * Latest stable [Rojo](https://github.com/rojo-rbx/rojo) * [Rokit](https://github.com/rojo-rbx/rokit) * [Luau Language Server](https://github.com/JohnnyMorganz/luau-lsp) (Only needed if working on the Studio plugin.) diff --git a/Cargo.lock b/Cargo.lock index 8ac0676b..0ffd1266 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1520,6 +1520,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -2068,6 +2074,7 @@ dependencies = [ "num_cpus", "opener", "paste", + "pathdiff", "pretty_assertions", "profiling", "rayon", @@ -2078,10 +2085,12 @@ dependencies = [ "rbx_xml", "reqwest", "ritz", + "rmp-serde", "roblox_install", "rojo-insta-ext", "semver", "serde", + "serde_bytes", "serde_json", "serde_yaml", "strum", @@ -2222,6 +2231,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_cbor" version = "0.11.2" diff --git a/Cargo.toml b/Cargo.toml index d8f34311..edf211a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -100,10 +100,13 @@ clap = { version = "3.2.25", features = ["derive"] } profiling = "1.0.15" yaml-rust2 = "0.10.3" data-encoding = "2.8.0" +pathdiff = "0.2.3" blake3 = "1.5.0" float-cmp = "0.9.0" indexmap = { version = "2.10.0", features = ["serde"] } +rmp-serde = "1.3.0" +serde_bytes = "0.11.19" [target.'cfg(windows)'.dependencies] winreg = "0.10.1" @@ -122,7 +125,7 @@ semver = "1.0.22" rojo-insta-ext = { path = "crates/rojo-insta-ext" } criterion = "0.3.6" -insta = { version = "1.36.1", features = ["redactions", "yaml"] } +insta = { version = "1.36.1", features = ["redactions", "yaml", "json"] } paste = "1.0.14" pretty_assertions = "1.4.0" serde_yaml = "0.8.26" diff --git a/build.rs b/build.rs index 6bfd8e55..d0457fb1 100644 --- a/build.rs +++ b/build.rs @@ -30,6 +30,11 @@ fn snapshot_from_fs_path(path: &Path) -> io::Result { continue; } + // Ignore images in msgpack-luau because they aren't UTF-8 encoded. + if file_name.ends_with(".png") { + continue; + } + let child_snapshot = snapshot_from_fs_path(&entry.path())?; children.push((file_name, child_snapshot)); } @@ -70,6 +75,7 @@ fn main() -> Result<(), anyhow::Error> { "src" => snapshot_from_fs_path(&plugin_dir.join("src"))?, "Packages" => snapshot_from_fs_path(&plugin_dir.join("Packages"))?, "Version.txt" => snapshot_from_fs_path(&plugin_dir.join("Version.txt"))?, + "UploadDetails.json" => snapshot_from_fs_path(&plugin_dir.join("UploadDetails.json"))?, }), }); diff --git a/plugin.project.json b/plugin.project.json index 83265250..1742547b 100644 --- a/plugin.project.json +++ b/plugin.project.json @@ -22,6 +22,9 @@ }, "Version": { "$path": "plugin/Version.txt" + }, + "UploadDetails": { + "$path": "plugin/UploadDetails.json" } } } diff --git a/plugin/Packages/msgpack-luau b/plugin/Packages/msgpack-luau new file mode 160000 index 00000000..40f67fc0 --- /dev/null +++ b/plugin/Packages/msgpack-luau @@ -0,0 +1 @@ +Subproject commit 40f67fc0f6eab20f8db9d3488d3d9f0928d1031b diff --git a/plugin/UploadDetails.json b/plugin/UploadDetails.json new file mode 100644 index 00000000..2396598f --- /dev/null +++ b/plugin/UploadDetails.json @@ -0,0 +1,7 @@ +{ + "assetId": 13916111004, + "name": "Rojo", + "description": "The plugin portion of Rojo, a tool to enable professional tooling for Roblox developers.", + "creatorId": 32644114, + "creatorType": "Group" +} \ No newline at end of file diff --git a/plugin/http/Response.lua b/plugin/http/Response.lua index 1230a1c9..b02ff644 100644 --- a/plugin/http/Response.lua +++ b/plugin/http/Response.lua @@ -1,5 +1,7 @@ local HttpService = game:GetService("HttpService") +local msgpack = require(script.Parent.Parent.msgpack) + local stringTemplate = [[ Http.Response { code: %d @@ -31,4 +33,8 @@ function Response:json() return HttpService:JSONDecode(self.body) end +function Response:msgpack() + return msgpack.decode(self.body) +end + return Response diff --git a/plugin/http/init.lua b/plugin/http/init.lua index cac0ebc4..3d05c6d3 100644 --- a/plugin/http/init.lua +++ b/plugin/http/init.lua @@ -1,7 +1,8 @@ local HttpService = game:GetService("HttpService") -local Promise = require(script.Parent.Promise) local Log = require(script.Parent.Log) +local msgpack = require(script.Parent.msgpack) +local Promise = require(script.Parent.Promise) local HttpError = require(script.Error) local HttpResponse = require(script.Response) @@ -13,6 +14,13 @@ local Http = {} Http.Error = HttpError Http.Response = HttpResponse +-- Monkey patch msgpack.UInt64.new to lossily convert the low and high bits of the integer +-- to a native Luau number. We should change the upstream decoder to emit a native +-- integer, once those are live. +function msgpack.UInt64.new(mostSignificantPart: number, leastSignificantPart: number): number + return (mostSignificantPart % 2 ^ 32) * 2 ^ 32 + (leastSignificantPart % 2 ^ 32) +end + local function performRequest(requestParams) local requestId = lastRequestId + 1 lastRequestId = requestId @@ -68,4 +76,12 @@ function Http.jsonDecode(source) return HttpService:JSONDecode(source) end +function Http.msgpackEncode(object) + return msgpack.encode(object) +end + +function Http.msgpackDecode(source) + return msgpack.decode(source) +end + return Http diff --git a/plugin/src/ApiContext.lua b/plugin/src/ApiContext.lua index 2d2459ec..2ce6ea14 100644 --- a/plugin/src/ApiContext.lua +++ b/plugin/src/ApiContext.lua @@ -145,7 +145,7 @@ function ApiContext:connect() return Http.get(url) :andThen(rejectFailedRequests) - :andThen(Http.Response.json) + :andThen(Http.Response.msgpack) :andThen(rejectWrongProtocolVersion) :andThen(function(body) assert(validateApiInfo(body)) @@ -163,7 +163,7 @@ end function ApiContext:read(ids) local url = ("%s/api/read/%s"):format(self.__baseUrl, table.concat(ids, ",")) - return Http.get(url):andThen(rejectFailedRequests):andThen(Http.Response.json):andThen(function(body) + return Http.get(url):andThen(rejectFailedRequests):andThen(Http.Response.msgpack):andThen(function(body) if body.sessionId ~= self.__sessionId then return Promise.reject("Server changed ID") end @@ -191,9 +191,9 @@ function ApiContext:write(patch) table.insert(updated, fixedUpdate) end - -- Only add the 'added' field if the table is non-empty, or else Roblox's - -- JSON implementation will turn the table into an array instead of an - -- object, causing API validation to fail. + -- Only add the 'added' field if the table is non-empty, or else the msgpack + -- encode implementation will turn the table into an array instead of a map, + -- causing API validation to fail. local added if next(patch.added) ~= nil then added = patch.added @@ -206,13 +206,16 @@ function ApiContext:write(patch) added = added, } - body = Http.jsonEncode(body) + body = Http.msgpackEncode(body) - return Http.post(url, body):andThen(rejectFailedRequests):andThen(Http.Response.json):andThen(function(responseBody) - Log.info("Write response: {:?}", responseBody) + return Http.post(url, body) + :andThen(rejectFailedRequests) + :andThen(Http.Response.msgpack) + :andThen(function(responseBody) + Log.info("Write response: {:?}", responseBody) - return responseBody - end) + return responseBody + end) end function ApiContext:connectWebSocket(packetHandlers) @@ -234,7 +237,7 @@ function ApiContext:connectWebSocket(packetHandlers) local closed, errored, received received = self.__wsClient.MessageReceived:Connect(function(msg) - local data = Http.jsonDecode(msg) + local data = Http.msgpackDecode(msg) if data.sessionId ~= self.__sessionId then Log.warn("Received message with wrong session ID; ignoring") return @@ -280,7 +283,7 @@ end function ApiContext:open(id) local url = ("%s/api/open/%s"):format(self.__baseUrl, id) - return Http.post(url, ""):andThen(rejectFailedRequests):andThen(Http.Response.json):andThen(function(body) + return Http.post(url, ""):andThen(rejectFailedRequests):andThen(Http.Response.msgpack):andThen(function(body) if body.sessionId ~= self.__sessionId then return Promise.reject("Server changed ID") end @@ -291,11 +294,11 @@ end function ApiContext:serialize(ids: { string }) local url = ("%s/api/serialize"):format(self.__baseUrl) - local request_body = Http.jsonEncode({ sessionId = self.__sessionId, ids = ids }) + local request_body = Http.msgpackEncode({ sessionId = self.__sessionId, ids = ids }) return Http.post(url, request_body) :andThen(rejectFailedRequests) - :andThen(Http.Response.json) + :andThen(Http.Response.msgpack) :andThen(function(response_body) if response_body.sessionId ~= self.__sessionId then return Promise.reject("Server changed ID") @@ -309,11 +312,11 @@ end function ApiContext:refPatch(ids: { string }) local url = ("%s/api/ref-patch"):format(self.__baseUrl) - local request_body = Http.jsonEncode({ sessionId = self.__sessionId, ids = ids }) + local request_body = Http.msgpackEncode({ sessionId = self.__sessionId, ids = ids }) return Http.post(url, request_body) :andThen(rejectFailedRequests) - :andThen(Http.Response.json) + :andThen(Http.Response.msgpack) :andThen(function(response_body) if response_body.sessionId ~= self.__sessionId then return Promise.reject("Server changed ID") diff --git a/plugin/src/Reconciler/diff.lua b/plugin/src/Reconciler/diff.lua index fb4c79d0..c8da0939 100644 --- a/plugin/src/Reconciler/diff.lua +++ b/plugin/src/Reconciler/diff.lua @@ -54,6 +54,10 @@ local function trueEquals(a, b): boolean end return true + -- For NaN, check if both values are not equal to themselves + elseif a ~= a and b ~= b then + return true + -- For numbers, compare with epsilon of 0.0001 to avoid floating point inequality elseif typeA == "number" and typeB == "number" then return fuzzyEq(a, b, 0.0001) diff --git a/rokit.toml b/rokit.toml index 6fbd8746..67dafa12 100644 --- a/rokit.toml +++ b/rokit.toml @@ -3,3 +3,4 @@ rojo = "rojo-rbx/rojo@7.5.1" selene = "Kampfkarren/selene@0.29.0" stylua = "JohnnyMorganz/stylua@2.1.0" run-in-roblox = "rojo-rbx/run-in-roblox@0.3.0" +lune = "lune-org/lune@0.10.4" diff --git a/src/cli/plugin.rs b/src/cli/plugin.rs index 03177654..af7025ca 100644 --- a/src/cli/plugin.rs +++ b/src/cli/plugin.rs @@ -98,5 +98,5 @@ fn uninstall_plugin() -> anyhow::Result<()> { #[test] fn plugin_initialize() { - assert!(initialize_plugin().is_ok()) + let _ = initialize_plugin().unwrap(); } diff --git a/src/cli/snapshots/librojo__cli__sourcemap__test__maps_absolute_paths.snap b/src/cli/snapshots/librojo__cli__sourcemap__test__maps_absolute_paths.snap new file mode 100644 index 00000000..33e9cf31 --- /dev/null +++ b/src/cli/snapshots/librojo__cli__sourcemap__test__maps_absolute_paths.snap @@ -0,0 +1,35 @@ +--- +source: src/cli/sourcemap.rs +expression: sourcemap_contents +--- +{ + "name": "default", + "className": "DataModel", + "filePaths": "[...1 path omitted...]", + "children": [ + { + "name": "ReplicatedStorage", + "className": "ReplicatedStorage", + "children": [ + { + "name": "Project", + "className": "ModuleScript", + "filePaths": "[...1 path omitted...]", + "children": [ + { + "name": "Module", + "className": "Folder", + "children": [ + { + "name": "module", + "className": "ModuleScript", + "filePaths": "[...1 path omitted...]" + } + ] + } + ] + } + ] + } + ] +} diff --git a/src/cli/snapshots/librojo__cli__sourcemap__test__maps_relative_paths.snap b/src/cli/snapshots/librojo__cli__sourcemap__test__maps_relative_paths.snap new file mode 100644 index 00000000..7a314b8c --- /dev/null +++ b/src/cli/snapshots/librojo__cli__sourcemap__test__maps_relative_paths.snap @@ -0,0 +1,41 @@ +--- +source: src/cli/sourcemap.rs +expression: sourcemap_contents +--- +{ + "name": "default", + "className": "DataModel", + "filePaths": [ + "default.project.json" + ], + "children": [ + { + "name": "ReplicatedStorage", + "className": "ReplicatedStorage", + "children": [ + { + "name": "Project", + "className": "ModuleScript", + "filePaths": [ + "src/init.luau" + ], + "children": [ + { + "name": "Module", + "className": "Folder", + "children": [ + { + "name": "module", + "className": "ModuleScript", + "filePaths": [ + "../module/module.luau" + ] + } + ] + } + ] + } + ] + } + ] +} diff --git a/src/cli/sourcemap.rs b/src/cli/sourcemap.rs index 95fe6631..94369e7c 100644 --- a/src/cli/sourcemap.rs +++ b/src/cli/sourcemap.rs @@ -10,7 +10,7 @@ use fs_err::File; use memofs::Vfs; use rayon::prelude::*; use rbx_dom_weak::{types::Ref, Ustr}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use tokio::runtime::Runtime; use crate::{ @@ -24,19 +24,20 @@ const PATH_STRIP_FAILED_ERR: &str = "Failed to create relative paths for project const ABSOLUTE_PATH_FAILED_ERR: &str = "Failed to turn relative path into absolute path!"; /// Representation of a node in the generated sourcemap tree. -#[derive(Serialize)] +#[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] struct SourcemapNode<'a> { name: &'a str, class_name: Ustr, #[serde( + default, skip_serializing_if = "Vec::is_empty", serialize_with = "crate::path_serializer::serialize_vec_absolute" )] file_paths: Vec>, - #[serde(skip_serializing_if = "Vec::is_empty")] + #[serde(default, skip_serializing_if = "Vec::is_empty")] children: Vec>, } @@ -70,12 +71,13 @@ pub struct SourcemapCommand { impl SourcemapCommand { pub fn run(self) -> anyhow::Result<()> { - let project_path = resolve_path(&self.project); + let project_path = fs_err::canonicalize(resolve_path(&self.project))?; - log::trace!("Constructing in-memory filesystem"); + log::trace!("Constructing filesystem with StdBackend"); let vfs = Vfs::new_default(); vfs.set_watch_enabled(self.watch); + log::trace!("Setting up session for sourcemap generation"); let session = ServeSession::new(vfs, project_path)?; let mut cursor = session.message_queue().cursor(); @@ -87,14 +89,17 @@ impl SourcemapCommand { // Pre-build a rayon threadpool with a low number of threads to avoid // dynamic creation overhead on systems with a high number of cpus. + log::trace!("Setting rayon global threadpool"); rayon::ThreadPoolBuilder::new() .num_threads(num_cpus::get().min(6)) .build_global() - .unwrap(); + .ok(); + log::trace!("Writing initial sourcemap"); write_sourcemap(&session, self.output.as_deref(), filter, self.absolute)?; if self.watch { + log::trace!("Setting up runtime for watch mode"); let rt = Runtime::new().unwrap(); loop { @@ -208,7 +213,7 @@ fn recurse_create_node<'a>( } else { for val in file_paths { output_file_paths.push(Cow::from( - val.strip_prefix(project_dir).expect(PATH_STRIP_FAILED_ERR), + pathdiff::diff_paths(val, project_dir).expect(PATH_STRIP_FAILED_ERR), )); } }; @@ -250,3 +255,80 @@ fn write_sourcemap( Ok(()) } + +#[cfg(test)] +mod test { + use crate::cli::sourcemap::SourcemapNode; + use crate::cli::SourcemapCommand; + use insta::internals::Content; + use std::path::Path; + + #[test] + fn maps_relative_paths() { + let sourcemap_dir = tempfile::tempdir().unwrap(); + let sourcemap_output = sourcemap_dir.path().join("sourcemap.json"); + let project_path = fs_err::canonicalize( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("test-projects") + .join("relative_paths") + .join("project"), + ) + .unwrap(); + let sourcemap_command = SourcemapCommand { + project: project_path, + output: Some(sourcemap_output.clone()), + include_non_scripts: false, + watch: false, + absolute: false, + }; + assert!(sourcemap_command.run().is_ok()); + + let raw_sourcemap_contents = fs_err::read_to_string(sourcemap_output.as_path()).unwrap(); + let sourcemap_contents = + serde_json::from_str::(&raw_sourcemap_contents).unwrap(); + insta::assert_json_snapshot!(sourcemap_contents); + } + + #[test] + fn maps_absolute_paths() { + let sourcemap_dir = tempfile::tempdir().unwrap(); + let sourcemap_output = sourcemap_dir.path().join("sourcemap.json"); + let project_path = fs_err::canonicalize( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("test-projects") + .join("relative_paths") + .join("project"), + ) + .unwrap(); + let sourcemap_command = SourcemapCommand { + project: project_path, + output: Some(sourcemap_output.clone()), + include_non_scripts: false, + watch: false, + absolute: true, + }; + assert!(sourcemap_command.run().is_ok()); + + let raw_sourcemap_contents = fs_err::read_to_string(sourcemap_output.as_path()).unwrap(); + let sourcemap_contents = + serde_json::from_str::(&raw_sourcemap_contents).unwrap(); + insta::assert_json_snapshot!(sourcemap_contents, { + ".**.filePaths" => insta::dynamic_redaction(|mut value, _path| { + let mut paths_count = 0; + + match value { + Content::Seq(ref mut vec) => { + for path in vec.iter().map(|i| i.as_str().unwrap()) { + assert_eq!(fs_err::canonicalize(path).is_ok(), true, "path was not valid"); + assert_eq!(Path::new(path).is_absolute(), true, "path was not absolute"); + + paths_count += 1; + } + } + _ => panic!("Expected filePaths to be a sequence"), + } + format!("[...{} path{} omitted...]", paths_count, if paths_count != 1 { "s" } else { "" } ) + }) + }); + } +} diff --git a/src/snapshot/patch_compute.rs b/src/snapshot/patch_compute.rs index cc3fbbbb..412c677a 100644 --- a/src/snapshot/patch_compute.rs +++ b/src/snapshot/patch_compute.rs @@ -8,7 +8,7 @@ use rbx_dom_weak::{ ustr, HashMapExt as _, UstrMap, UstrSet, }; -use crate::{RojoRef, REF_POINTER_ATTRIBUTE_PREFIX}; +use crate::{variant_eq::variant_eq, RojoRef, REF_POINTER_ATTRIBUTE_PREFIX}; use super::{ patch::{PatchAdd, PatchSet, PatchUpdate}, @@ -127,7 +127,7 @@ fn compute_property_patches( match instance.properties().get(&name) { Some(instance_value) => { - if &snapshot_value != instance_value { + if !variant_eq(&snapshot_value, instance_value) { changed_properties.insert(name, Some(snapshot_value)); } } diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 61d26674..5dc3028f 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -301,6 +301,7 @@ pub fn get_best_middleware(snapshot: &SyncbackSnapshot) -> Middleware { static JSON_MODEL_CLASSES: OnceLock> = OnceLock::new(); let json_model_classes = JSON_MODEL_CLASSES.get_or_init(|| { [ + "Actor", "Sound", "SoundGroup", "Sky", @@ -318,6 +319,11 @@ pub fn get_best_middleware(snapshot: &SyncbackSnapshot) -> Middleware { "ChatInputBarConfiguration", "BubbleChatConfiguration", "ChannelTabsConfiguration", + "RemoteEvent", + "UnreliableRemoteEvent", + "RemoteFunction", + "BindableEvent", + "BindableFunction", ] .into() }); diff --git a/src/web/api.rs b/src/web/api.rs index 5fbd7b6d..38163bb9 100644 --- a/src/web/api.rs +++ b/src/web/api.rs @@ -13,7 +13,6 @@ use rbx_dom_weak::{ }; use crate::{ - json, serve_session::ServeSession, snapshot::{InstanceWithMeta, PatchSet, PatchUpdate}, web::{ @@ -22,11 +21,10 @@ use crate::{ ServerInfoResponse, SocketPacket, SocketPacketBody, SocketPacketType, SubscribeMessage, WriteRequest, WriteResponse, PROTOCOL_VERSION, SERVER_VERSION, }, - util::{json, json_ok}, + util::{deserialize_msgpack, msgpack, msgpack_ok, serialize_msgpack}, }, web_api::{ - BufferEncode, InstanceUpdate, RefPatchRequest, RefPatchResponse, SerializeRequest, - SerializeResponse, + InstanceUpdate, RefPatchRequest, RefPatchResponse, SerializeRequest, SerializeResponse, }, }; @@ -42,7 +40,7 @@ pub async fn call(serve_session: Arc, mut request: Request) if is_upgrade_request(&request) { service.handle_api_socket(&mut request).await } else { - json( + msgpack( ErrorResponse::bad_request( "/api/socket must be called as a websocket upgrade request", ), @@ -58,7 +56,7 @@ pub async fn call(serve_session: Arc, mut request: Request) } (&Method::POST, "/api/write") => service.handle_api_write(request).await, - (_method, path) => json( + (_method, path) => msgpack( ErrorResponse::not_found(format!("Route not found: {}", path)), StatusCode::NOT_FOUND, ), @@ -79,7 +77,7 @@ impl ApiService { let tree = self.serve_session.tree(); let root_instance_id = tree.get_root_id(); - json_ok(&ServerInfoResponse { + msgpack_ok(&ServerInfoResponse { server_version: SERVER_VERSION.to_owned(), protocol_version: PROTOCOL_VERSION, session_id: self.serve_session.session_id(), @@ -98,7 +96,7 @@ impl ApiService { let input_cursor: u32 = match argument.parse() { Ok(v) => v, Err(err) => { - return json( + return msgpack( ErrorResponse::bad_request(format!("Malformed message cursor: {}", err)), StatusCode::BAD_REQUEST, ); @@ -109,7 +107,7 @@ impl ApiService { let (response, websocket) = match upgrade(request, None) { Ok(result) => result, Err(err) => { - return json( + return msgpack( ErrorResponse::internal_error(format!("WebSocket upgrade failed: {}", err)), StatusCode::INTERNAL_SERVER_ERROR, ); @@ -136,10 +134,10 @@ impl ApiService { let body = body::to_bytes(request.into_body()).await.unwrap(); - let request: WriteRequest = match json::from_slice(&body) { + let request: WriteRequest = match deserialize_msgpack(&body) { Ok(request) => request, Err(err) => { - return json( + return msgpack( ErrorResponse::bad_request(format!("Invalid body: {}", err)), StatusCode::BAD_REQUEST, ); @@ -147,7 +145,7 @@ impl ApiService { }; if request.session_id != session_id { - return json( + return msgpack( ErrorResponse::bad_request("Wrong session ID"), StatusCode::BAD_REQUEST, ); @@ -173,7 +171,7 @@ impl ApiService { }) .unwrap(); - json_ok(WriteResponse { session_id }) + msgpack_ok(WriteResponse { session_id }) } async fn handle_api_read(&self, request: Request) -> Response { @@ -183,7 +181,7 @@ impl ApiService { let requested_ids = match requested_ids { Ok(ids) => ids, Err(_) => { - return json( + return msgpack( ErrorResponse::bad_request("Malformed ID list"), StatusCode::BAD_REQUEST, ); @@ -207,7 +205,7 @@ impl ApiService { } } - json_ok(ReadResponse { + msgpack_ok(ReadResponse { session_id: self.serve_session.session_id(), message_cursor, instances, @@ -225,10 +223,10 @@ impl ApiService { let session_id = self.serve_session.session_id(); let body = body::to_bytes(request.into_body()).await.unwrap(); - let request: SerializeRequest = match json::from_slice(&body) { + let request: SerializeRequest = match deserialize_msgpack(&body) { Ok(request) => request, Err(err) => { - return json( + return msgpack( ErrorResponse::bad_request(format!("Invalid body: {}", err)), StatusCode::BAD_REQUEST, ); @@ -236,7 +234,7 @@ impl ApiService { }; if request.session_id != session_id { - return json( + return msgpack( ErrorResponse::bad_request("Wrong session ID"), StatusCode::BAD_REQUEST, ); @@ -269,7 +267,7 @@ impl ApiService { response_dom.transfer_within(child_ref, object_value); } else { - json( + msgpack( ErrorResponse::bad_request(format!("provided id {id} is not in the tree")), StatusCode::BAD_REQUEST, ); @@ -280,9 +278,9 @@ impl ApiService { let mut source = Vec::new(); rbx_binary::to_writer(&mut source, &response_dom, &[response_dom.root_ref()]).unwrap(); - json_ok(SerializeResponse { + msgpack_ok(SerializeResponse { session_id: self.serve_session.session_id(), - model_contents: BufferEncode::new(source), + model_contents: source, }) } @@ -294,10 +292,10 @@ impl ApiService { let session_id = self.serve_session.session_id(); let body = body::to_bytes(request.into_body()).await.unwrap(); - let request: RefPatchRequest = match json::from_slice(&body) { + let request: RefPatchRequest = match deserialize_msgpack(&body) { Ok(request) => request, Err(err) => { - return json( + return msgpack( ErrorResponse::bad_request(format!("Invalid body: {}", err)), StatusCode::BAD_REQUEST, ); @@ -305,7 +303,7 @@ impl ApiService { }; if request.session_id != session_id { - return json( + return msgpack( ErrorResponse::bad_request("Wrong session ID"), StatusCode::BAD_REQUEST, ); @@ -338,7 +336,7 @@ impl ApiService { } } - json_ok(RefPatchResponse { + msgpack_ok(RefPatchResponse { session_id: self.serve_session.session_id(), patch: SubscribeMessage { added: HashMap::new(), @@ -354,7 +352,7 @@ impl ApiService { let requested_id = match Ref::from_str(argument) { Ok(id) => id, Err(_) => { - return json( + return msgpack( ErrorResponse::bad_request("Invalid instance ID"), StatusCode::BAD_REQUEST, ); @@ -366,7 +364,7 @@ impl ApiService { let instance = match tree.get_instance(requested_id) { Some(instance) => instance, None => { - return json( + return msgpack( ErrorResponse::bad_request("Instance not found"), StatusCode::NOT_FOUND, ); @@ -376,7 +374,7 @@ impl ApiService { let script_path = match pick_script_path(instance) { Some(path) => path, None => { - return json( + return msgpack( ErrorResponse::bad_request( "No appropriate file could be found to open this script", ), @@ -389,7 +387,7 @@ impl ApiService { Ok(()) => {} Err(error) => match error { OpenError::Io(io_error) => { - return json( + return msgpack( ErrorResponse::internal_error(format!( "Attempting to open {} failed because of the following io error: {}", script_path.display(), @@ -403,7 +401,7 @@ impl ApiService { status, stderr, } => { - return json( + return msgpack( ErrorResponse::internal_error(format!( r#"The command '{}' to open '{}' failed with the error code '{}'. Error logs: @@ -419,7 +417,7 @@ impl ApiService { }, }; - json_ok(OpenResponse { + msgpack_ok(OpenResponse { session_id: self.serve_session.session_id(), }) } @@ -483,7 +481,7 @@ async fn handle_websocket_subscription( match result { Ok((new_cursor, messages)) => { if !messages.is_empty() { - let json_message = { + let msgpack_message = { let tree = tree_handle.lock().unwrap(); let api_messages = messages .into_iter() @@ -499,12 +497,12 @@ async fn handle_websocket_subscription( }), }; - serde_json::to_string(&response)? + serialize_msgpack(response)? }; log::debug!("Sending batch of messages over WebSocket subscription"); - if websocket.send(Message::Text(json_message)).await.is_err() { + if websocket.send(Message::Binary(msgpack_message)).await.is_err() { // Client disconnected log::debug!("WebSocket subscription closed by client"); break; diff --git a/src/web/interface.rs b/src/web/interface.rs index 430f13ec..51307b67 100644 --- a/src/web/interface.rs +++ b/src/web/interface.rs @@ -249,31 +249,8 @@ pub struct SerializeRequest { #[serde(rename_all = "camelCase")] pub struct SerializeResponse { pub session_id: SessionId, - pub model_contents: BufferEncode, -} - -/// Using this struct we can force Roblox to JSONDecode this as a buffer. -/// This is what Roblox's serde APIs use, so it saves a step in the plugin. -#[derive(Debug, Serialize, Deserialize)] -pub struct BufferEncode { - m: (), - t: Cow<'static, str>, - base64: String, -} - -impl BufferEncode { - pub fn new(content: Vec) -> Self { - let base64 = data_encoding::BASE64.encode(&content); - Self { - m: (), - t: Cow::Borrowed("buffer"), - base64, - } - } - - pub fn model(&self) -> &str { - &self.base64 - } + #[serde(with = "serde_bytes")] + pub model_contents: Vec, } #[derive(Debug, Serialize, Deserialize)] diff --git a/src/web/util.rs b/src/web/util.rs index 2b6882e5..1c1cbb50 100644 --- a/src/web/util.rs +++ b/src/web/util.rs @@ -1,8 +1,48 @@ use hyper::{header::CONTENT_TYPE, Body, Response, StatusCode}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; -pub fn json_ok(value: T) -> Response { - json(value, StatusCode::OK) +pub fn msgpack_ok(value: T) -> Response { + msgpack(value, StatusCode::OK) +} + +pub fn msgpack(value: T, code: StatusCode) -> Response { + let mut serialized = Vec::new(); + let mut serializer = rmp_serde::Serializer::new(&mut serialized) + .with_human_readable() + .with_struct_map(); + + if let Err(err) = value.serialize(&mut serializer) { + return Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .header(CONTENT_TYPE, "text/plain") + .body(Body::from(err.to_string())) + .unwrap(); + }; + + Response::builder() + .status(code) + .header(CONTENT_TYPE, "application/msgpack") + .body(Body::from(serialized)) + .unwrap() +} + +pub fn serialize_msgpack(value: T) -> anyhow::Result> { + let mut serialized = Vec::new(); + let mut serializer = rmp_serde::Serializer::new(&mut serialized) + .with_human_readable() + .with_struct_map(); + + value.serialize(&mut serializer)?; + + Ok(serialized) +} + +pub fn deserialize_msgpack<'a, T: Deserialize<'a>>( + input: &'a [u8], +) -> Result { + let mut deserializer = rmp_serde::Deserializer::new(input).with_human_readable(); + + T::deserialize(&mut deserializer) } pub fn json(value: T, code: StatusCode) -> Response { diff --git a/test-projects/relative_paths/default.project.json b/test-projects/relative_paths/default.project.json new file mode 100644 index 00000000..af85c143 --- /dev/null +++ b/test-projects/relative_paths/default.project.json @@ -0,0 +1,14 @@ +{ + "name": "default", + "tree": { + "$className": "DataModel", + "ReplicatedStorage": { + "Project": { + "$path": "project/src", + "Module": { + "$path": "module" + } + } + } + } +} diff --git a/test-projects/relative_paths/module/module.luau b/test-projects/relative_paths/module/module.luau new file mode 100644 index 00000000..15e53c4a --- /dev/null +++ b/test-projects/relative_paths/module/module.luau @@ -0,0 +1 @@ +return nil diff --git a/test-projects/relative_paths/project/default.project.json b/test-projects/relative_paths/project/default.project.json new file mode 100644 index 00000000..a7d8d46c --- /dev/null +++ b/test-projects/relative_paths/project/default.project.json @@ -0,0 +1,14 @@ +{ + "name": "default", + "tree": { + "$className": "DataModel", + "ReplicatedStorage": { + "Project": { + "$path": "src/", + "Module": { + "$path": "../module" + } + } + } + } +} diff --git a/test-projects/relative_paths/project/src/init.luau b/test-projects/relative_paths/project/src/init.luau new file mode 100644 index 00000000..15e53c4a --- /dev/null +++ b/test-projects/relative_paths/project/src/init.luau @@ -0,0 +1 @@ +return nil diff --git a/tests/rojo_test/serve_util.rs b/tests/rojo_test/serve_util.rs index a5decdbb..4b67a883 100644 --- a/tests/rojo_test/serve_util.rs +++ b/tests/rojo_test/serve_util.rs @@ -10,6 +10,7 @@ use std::{ use hyper_tungstenite::tungstenite::{connect, Message}; use rbx_dom_weak::types::Ref; +use serde::{Deserialize, Serialize}; use tempfile::{tempdir, TempDir}; use librojo::{ @@ -161,22 +162,16 @@ impl TestServeSession { pub fn get_api_rojo(&self) -> Result { let url = format!("http://localhost:{}/api/rojo", self.port); - let body = reqwest::blocking::get(url)?.text()?; + let body = reqwest::blocking::get(url)?.bytes()?; - let value = jsonc_parser::parse_to_serde_value(&body, &Default::default()) - .expect("Failed to parse JSON") - .expect("No JSON value"); - Ok(serde_json::from_value(value).expect("Server returned malformed response")) + Ok(deserialize_msgpack(&body).expect("Server returned malformed response")) } pub fn get_api_read(&self, id: Ref) -> Result, reqwest::Error> { let url = format!("http://localhost:{}/api/read/{}", self.port, id); - let body = reqwest::blocking::get(url)?.text()?; + let body = reqwest::blocking::get(url)?.bytes()?; - let value = jsonc_parser::parse_to_serde_value(&body, &Default::default()) - .expect("Failed to parse JSON") - .expect("No JSON value"); - Ok(serde_json::from_value(value).expect("Server returned malformed response")) + Ok(deserialize_msgpack(&body).expect("Server returned malformed response")) } pub fn get_api_socket_packet( @@ -198,8 +193,8 @@ impl TestServeSession { } match socket.read() { - Ok(Message::Text(text)) => { - let packet: SocketPacket = serde_json::from_str(&text)?; + Ok(Message::Binary(binary)) => { + let packet: SocketPacket = deserialize_msgpack(&binary)?; if packet.packet_type != packet_type { continue; } @@ -212,7 +207,7 @@ impl TestServeSession { return Err("WebSocket closed before receiving messages".into()); } Ok(_) => { - // Ignore other message types (ping, pong, binary) + // Ignore other message types (ping, pong, text) continue; } Err(hyper_tungstenite::tungstenite::Error::Io(e)) @@ -236,15 +231,37 @@ impl TestServeSession { ) -> Result { let client = reqwest::blocking::Client::new(); let url = format!("http://localhost:{}/api/serialize", self.port); - let body = serde_json::to_string(&SerializeRequest { + let body = serialize_msgpack(&SerializeRequest { session_id, ids: ids.to_vec(), - }); + }) + .unwrap(); - client.post(url).body((body).unwrap()).send()?.json() + let body = client.post(url).body(body).send()?.bytes()?; + + Ok(deserialize_msgpack(&body).expect("Server returned malformed response")) } } +fn serialize_msgpack(value: T) -> Result, rmp_serde::encode::Error> { + let mut serialized = Vec::new(); + let mut serializer = rmp_serde::Serializer::new(&mut serialized) + .with_human_readable() + .with_struct_map(); + + value.serialize(&mut serializer)?; + + Ok(serialized) +} + +fn deserialize_msgpack<'a, T: Deserialize<'a>>( + input: &'a [u8], +) -> Result { + let mut deserializer = rmp_serde::Deserializer::new(input).with_human_readable(); + + T::deserialize(&mut deserializer) +} + /// Probably-okay way to generate random enough port numbers for running the /// Rojo live server. /// @@ -262,11 +279,7 @@ fn get_port_number() -> usize { /// Since the provided structure intentionally includes unredacted referents, /// some post-processing is done to ensure they don't show up in the model. pub fn serialize_to_xml_model(response: &SerializeResponse, redactions: &RedactionMap) -> String { - let model_content = data_encoding::BASE64 - .decode(response.model_contents.model().as_bytes()) - .unwrap(); - - let mut dom = rbx_binary::from_reader(model_content.as_slice()).unwrap(); + let mut dom = rbx_binary::from_reader(response.model_contents.as_slice()).unwrap(); // This makes me realize that maybe we need a `descendants_mut` iter. let ref_list: Vec = dom.descendants().map(|inst| inst.referent()).collect(); for referent in ref_list {