feat: add serializeArrays option (#29)

* feat: add optional `convertArrays`

* perf: sync atomic changes to keys

* feat: add `serializeArrays` option

* docs: rephrase debug mode flag

.prettierrc

@@ -1,6 +1,6 @@
 {
 	"printWidth": 120,
-  "tabWidth": 2,
+	"tabWidth": 4,
 	"trailingComma": "all",
 	"useTabs": true
 }

README.md

@@ -55,7 +55,7 @@ Charm = "littensy/charm@VERSION"
 
 ## 🐛 Debugging
 
-Charm provides a debug mode to help you identify potential bugs in your project. To enable debug mode, set the global `_G.__DEV__` flag to `true` at the entry point of your project.
+Charm provides a debug mode to help you identify potential bugs in your project. To enable debug mode, set the global `_G.__DEV__` flag to `true` before importing Charm.
 
 Enabling `__DEV__` adds a few helpful features:
 
@@ -441,6 +441,7 @@ local syncer = CharmSync.server({
 	atoms = atomsToSync,     -- A dictionary of the atoms to sync, matching the client's
 	interval = 0,            -- The minimum interval between state updates
 	preserveHistory = false, -- Whether to send a full history of changes made to the atoms (slower)
+	serializeArrays = true,  -- Safety measures for arrays, should be false when using ByteNet or Zap
 })
 
 -- Sends state updates to clients when a synced atom changes.
@@ -466,6 +467,8 @@ end)
 
     -   **optional** `preserveHistory`: Whether to sync an exhaustive history of changes made to the atoms since the last sync event. If `true`, the server sends multiple payloads instead of one. Defaults to `false` for performance.
 
+    -   **optional** `serializeArrays`: Whether to convert problematic arrays to dictionaries before firing remote events. Because arrays in state patches can have holes, extra work is required to send them over a remote event intact. Defaults to `true`, but should be `false` if payloads are serialized (i.e. if you use [ByteNet](https://github.com/ffrostfall/ByteNet) or [Zap](https://github.com/red-blox/zap)).
+
 #### Returns
 
 `server` returns an object with the following methods:
@@ -476,9 +479,11 @@ end)
 
 #### Caveats
 
-- By default, Charm omits the individual changes made to atoms between sync events (i.e. a `counterAtom` set to `1` and then `2` will only send the final state of `2`). If you need to preserve a history of changes, set `preserveHistory` to `true`.
+-   **Do not use values that cannot be sent over remotes** in your shared atoms. This includes functions, threads, and non-string keys in dictionaries.
 
-- Charm does not handle network communication. You must implement your own network layer to send and receive state updates. This is implemented via the `remotes` namespace in the example above.
+-   **By default, Charm omits the individual changes made to atoms** between sync events (i.e. a `counterAtom` set to `1` and then `2` will only send the final state of `2`). If you need to preserve a history of changes, set `preserveHistory` to `true`.
+
+-   **Charm does not handle network communication.** Use remote events or a network library to send sync payloads - and remember to set `serializeArrays` accordingly!
 
 ---
 
@@ -489,6 +494,7 @@ Call `client` to create a client sync object. This synchronizes the client's ato
 ```luau
 local syncer = CharmSync.client({
 	atoms = atomsToSync, -- A dictionary of the atoms to sync, matching the server's
+	ignoreUnhydrated = true, -- Whether to ignore state updates before the initial update
 })
 
 -- Applies state updates from the server to the client's atoms.
@@ -508,6 +514,8 @@ remotes.requestState:fire()
 
     -   `atoms`: A dictionary of the atoms to sync. The keys should match the keys on the server.
 
+    -   **optional** `ignoreUnhydrated`: Whether to ignore state updates before setting the initial state. Defaults to `true`.
+
 #### Returns
 
 `client` returns an object with the following methods:
@@ -516,7 +524,7 @@ remotes.requestState:fire()
 
 #### Caveats
 
-- The client sync object does not handle network communication. You must implement your own network layer to send and receive state updates. This includes requesting the initial state, which is implemented via `requestState` in the example above.
+-   **Charm does not handle network communication.** Use remote events or a network library to receive sync payloads. This includes requesting the initial state, which is implemented via `requestState` in the example above.
 
 ---
 

default.project.json

@@ -23,7 +23,7 @@
 		"ServerScriptService": {
 			"$className": "ServerScriptService",
 			"Tests": {
-				"$path": "Tests"
+				"$path": "tests"
 			}
 		}
 	}

src/charm-sync/src/client.luau

@@ -7,7 +7,7 @@ type ClientOptions = {
 	--[=[
 		The atoms to synchronize with the server.
 	]=]
-	atoms: { [string]: Charm.Atom<any> },
+	atoms: { [string | number]: Charm.Atom<any> },
 	--[=[
 		Whether to ignore patches sent before the client has been hydrated.
 		Default is `true`.
@@ -33,33 +33,44 @@ type ClientSyncer = {
 	@return A `ClientSyncer` object.
 ]=]
 local function client(options: ClientOptions): ClientSyncer
+	local self = {} :: ClientSyncer
+
 	local atoms = options.atoms
 	local ignoreUnhydrated = options.ignoreUnhydrated ~= false
-	local self = {} :: ClientSyncer
 	local hydrated = false
 
-	-- Apply the state changes sent by the server.
+	local function hydrate(state: { [string | number]: any })
+		hydrated = true
+
+		for key, value in next, state do
+			local atom = atoms[key]
+			atom(value)
+		end
+	end
+
+	local function apply(data: { [string | number]: any })
+		local target = {}
+
+		for key, atom in next, atoms do
+			target[key] = atom()
+		end
+
+		target = patch.apply(target, data)
+
+		for key, atom in next, atoms do
+			atom(target[key])
+		end
+	end
+
 	function self:sync(...)
 		for index = 1, select("#", ...) do
 			local payload: SyncPayload = select(index, ...)
 
-			if ignoreUnhydrated and payload.type == "patch" and not hydrated then
-				-- If the client is not initialized, ignore patches. This is
-				-- to prevent patching incomplete state.
-				continue
-			end
-
-			hydrated = true
-
 			Charm.batch(function()
-				for key, state in next, payload.data do
+				if payload.type == "init" then
-					local atom = atoms[key]
+					hydrate(payload.data)
-
+				elseif not ignoreUnhydrated or hydrated then
-					if payload.type == "patch" then
+					apply(payload.data)
-						atom(patch.apply(atom(), state))
-					else
-						atom(state)
-					end
 				end
 			end)
 		end

src/charm-sync/src/index.d.ts

@@ -8,23 +8,22 @@ type Cleanup = () => void;
 declare namespace CharmSync {
 	type AtomMap = Record<string, Atom<any>>;
 
+	type SelectorMap = Record<string, Selector<any>>;
+
 	/**
 	 * @deprecated Use `SelectorMap` instead.
 	 */
-	type MoleculeMap = Record<string, Selector<any>>;
+	type MoleculeMap = SelectorMap;
-
-	type SelectorMap = Record<string, Selector<any>>;
 
 	/**
 	 * Infers the type of the return values produced by a map of functions.
 	 */
 	type StateOfMap<T> = {
-		[P in keyof T]: T[P] extends Selector<infer State> ? State : never;
+		readonly [P in keyof T]: T[P] extends Selector<infer State> ? State : never;
 	};
 
 	/**
-	 * A special value that denotes the absence of a value. Used to represent
+	 * Represents the removal of a value from the state.
-	 * removed values in patches.
 	 */
 	interface None {
 		readonly __none: "__none";
@@ -96,8 +95,8 @@ declare namespace CharmSync {
 	 * state between the two.
 	 */
 	type SyncPayload<Selectors extends SelectorMap> =
-		| { type: "init"; data: StateOfMap<Selectors> }
+		| { readonly type: "init"; readonly data: StateOfMap<Selectors> }
-		| { type: "patch"; data: SyncPatch<StateOfMap<Selectors>> };
+		| { readonly type: "patch"; readonly data: SyncPatch<StateOfMap<Selectors>> };
 
 	interface ClientOptions<Atoms extends AtomMap> {
 		/**
@@ -132,6 +131,15 @@ declare namespace CharmSync {
 		 * to reconstruct the state's changes over time.
 		 */
 		preserveHistory?: boolean;
+		/**
+		 * When `true`, convert problematic sparse arrays into dictionaries
+		 * with string keys. Defaults to `true`.
+		 *
+		 * While this is mandatory for safely syncing arrays over vanilla Roblox
+		 * remotes, it can be disabled if your network library uses a custom
+		 * serialization method (i.e. Zap, ByteNet).
+		 */
+		serializeArrays?: boolean;
 	}
 
 	interface ClientSyncer<Atoms extends AtomMap> {

src/charm-sync/src/patch.luau

@@ -1,5 +1,7 @@
 local validate = require(script.Parent.validate)
 
+local __DEV__ = _G.__DEV__
+
 --[=[
 	A special symbol that denotes the absence of a value. Used to represent
 	deleted values in patches.
@@ -37,7 +39,16 @@ local function stringifySparseArray(object: { any }): { [string | number]: any }
 	return dictionary
 end
 
-local function diff(prevState: { [any]: any }, nextState: { [any]: any })
+--[=[
+	Compares two states and returns a patch that can be applied to the previous
+	state to produce the next state.
+
+	@param prevState The previous state.
+	@param nextState The next state.
+	@param serializeArrays Whether to convert sparse arrays into dictionaries. Defaults to `true`.
+	@returns A patch that can be applied to the previous state.
+]=]
+local function diff(prevState: { [any]: any }, nextState: { [any]: any }, serializeArrays: boolean?)
 	local patches = table.clone(nextState)
 
 	for key, previous in next, prevState do
@@ -48,16 +59,16 @@ local function diff(prevState: { [any]: any }, nextState: { [any]: any })
 		elseif current == nil then
 			patches[key] = None
 		elseif type(previous) == "table" and type(current) == "table" then
-			patches[key] = diff(previous, current)
+			patches[key] = diff(previous, current, serializeArrays)
 		end
 	end
 
 	-- Coerce sparse array patches into dictionaries
-	if prevState[1] ~= nil or nextState[1] ~= nil then
+	if serializeArrays ~= false and (prevState[1] ~= nil or nextState[1] ~= nil) then
 		patches = stringifySparseArray(patches)
 	end
 
-	if _G.__DEV__ then
+	if __DEV__ then
 		for key, value in next, prevState do
 			validate(value, key)
 		end
@@ -72,6 +83,13 @@ local function diff(prevState: { [any]: any }, nextState: { [any]: any })
 	return patches
 end
 
+--[=[
+	Applies a patch to a state and returns the next state.
+
+	@param state The current state.
+	@param patches The patches to apply.
+	@returns The next state.
+]=]
 local function apply(state: any, patches: any): any
 	if type(patches) == "table" and patches.__none == "__none" then
 		return nil
@@ -97,7 +115,6 @@ local function apply(state: any, patches: any): any
 end
 
 return {
-	NONE = None,
 	isNone = isNone,
 	diff = diff,
 	apply = apply,

src/charm-sync/src/server.luau

@@ -10,7 +10,7 @@ type ServerOptions = {
 	--[=[
 		The atoms to synchronize with the client.
 	]=]
-	atoms: { [string]: Charm.Atom<any> },
+	atoms: { [string | number]: Charm.Atom<any> },
 	--[=[
 		The interval at which to send patches to the client, in seconds.
 		Defaults to `0` (patches are sent up to once per frame). Set to a
@@ -23,6 +23,15 @@ type ServerOptions = {
 		per frame, where each individual change is important. Defaults to `false`.
 	]=]
 	preserveHistory: boolean?,
+	--[=[
+		Whether to convert problematic sparse arrays into dictionaries with
+		string keys. Defaults to `true`.
+
+		While this is mandatory for safely syncing arrays over vanilla Roblox
+		remotes, it can be disabled if your network library uses a custom
+		serialization method (i.e. Zap, ByteNet).
+	]=]
+	serializeArrays: boolean?,
 }
 
 type ServerSyncer = {
@@ -63,7 +72,7 @@ local function server(options: ServerOptions): ServerSyncer
 	local sync: (player: Player, payload: SyncPayload) -> ()
 
 	local function createSnapshot()
-		local snapshot: { [string]: any } = {}
+		local snapshot = {}
 
 		for key, atom in next, options.atoms do
 			snapshot[key] = atom()
@@ -72,10 +81,20 @@ local function server(options: ServerOptions): ServerSyncer
 		return snapshot
 	end
 
+	-- Send the initial state to a player when they join the server.
+	function self:hydrate(player)
+		assert(sync, "connect() must be called before hydrate()")
+
+		sync(player, {
+			type = "init",
+			data = createSnapshot(),
+		})
+	end
+
+	-- If the history of state changes should be preserved, keep a list of
+	-- previous snapshots and diff-check each one from the last.
 	if options.preserveHistory then
-		-- Preserve a history of every state change to allow for more granular
+		local snapshots = { createSnapshot() }
-		-- syncing with the client.
-		local snapshots: { { [string]: any } } = {}
 		local changed = false
 
 		function self:connect(callback)
@@ -83,7 +102,7 @@ local function server(options: ServerOptions): ServerSyncer
 
 			sync = callback
 
-			local function pushSnapshot(key: string, current: any, previous: any)
+			local function pushSnapshot(key: string | number, current: any, previous: any)
 				local lastSnapshot = snapshots[#snapshots]
 				local previousSnapshot = snapshots[#snapshots - 1]
 
@@ -108,8 +127,6 @@ local function server(options: ServerOptions): ServerSyncer
 				end)
 			end
 
-			snapshots[1] = createSnapshot()
-
 			local disconnect = interval(function()
 				if not changed then
 					return
@@ -127,7 +144,7 @@ local function server(options: ServerOptions): ServerSyncer
 
 					table.insert(payloads, {
 						type = "patch",
-						data = patch.diff(snapshots[index - 1], snapshot),
+						data = patch.diff(snapshots[index - 1], snapshot, options.serializeArrays),
 					})
 				end
 
@@ -164,7 +181,7 @@ local function server(options: ServerOptions): ServerSyncer
 
 				table.insert(payloads, {
 					type = "patch",
-					data = patch.diff(snapshots[index - 1], snapshot),
+					data = patch.diff(snapshots[index - 1], snapshot, options.serializeArrays),
 				})
 			end
 
@@ -173,8 +190,10 @@ local function server(options: ServerOptions): ServerSyncer
 
 			sync(player, unpack(payloads))
 		end
-	else
+
-		-- Only send the most recent state to the client.
+		return self
+	end
+
 	local previousSnapshot = createSnapshot()
 	local currentSnapshot = table.clone(previousSnapshot)
 	local changed = false
@@ -199,7 +218,7 @@ local function server(options: ServerOptions): ServerSyncer
 
 			local payload: SyncPayload = {
 				type = "patch",
-					data = patch.diff(previousSnapshot, currentSnapshot),
+				data = patch.diff(previousSnapshot, currentSnapshot, options.serializeArrays),
 			}
 
 			previousSnapshot = table.clone(currentSnapshot)
@@ -219,7 +238,7 @@ local function server(options: ServerOptions): ServerSyncer
 		end
 	end
 
-		function self:_sendPatch(player: Player)
+	function self:_sendPatch(player)
 		assert(sync, "connect() must be called before _sendPatch()")
 
 		if not changed then
@@ -228,7 +247,7 @@ local function server(options: ServerOptions): ServerSyncer
 
 		local payload: SyncPayload = {
 			type = "patch",
-				data = patch.diff(previousSnapshot, currentSnapshot),
+			data = patch.diff(previousSnapshot, currentSnapshot, options.serializeArrays),
 		}
 
 		previousSnapshot = table.clone(currentSnapshot)
@@ -236,17 +255,6 @@ local function server(options: ServerOptions): ServerSyncer
 
 		sync(player, payload)
 	end
-	end
-
-	-- Send the initial state to a player when they join the server.
-	function self:hydrate(player)
-		assert(sync, "connect() must be called before hydrate()")
-
-		sync(player, {
-			type = "init",
-			data = createSnapshot(),
-		})
-	end
 
 	return self
 end

src/charm-sync/src/types.luau

@@ -3,8 +3,11 @@
 	state between the two.
 ]=]
 export type SyncPayload = {
-	type: "patch" | "init",
+	type: "init",
-	data: { [any]: any },
+	data: { [string | number]: any },
+} | {
+	type: "patch",
+	data: { [string | number]: any },
 }
 
 return nil

src/charm-sync/src/validate.luau

@@ -1,9 +1,7 @@
-local Charm = require(script.Parent.Parent.Charm)
-
 local SAFE_KEYS = { string = true, number = true }
-local SAFE_VALUES = { string = true, number = true, boolean = true, table = true, userdata = true }
+local SAFE_VALUES = { string = true, number = true, boolean = true, table = true, userdata = true, vector = true }
 
-local function isUnsafeTable(object: { [any]: any })
+local function isTableUnsafe(object: { [any]: any })
 	local keyType = nil
 	local objectSize = 0
 
@@ -47,9 +45,7 @@ local function validate(value: any, key: any)
 	elseif typeOfValue == "table" then
 		if getmetatable(value) ~= nil then
 			error(`Cannot sync tables with metatables! Got {value} at key '{key}'`)
-		elseif Charm.isAtom(value) then
+		elseif isTableUnsafe(value) then
-			error(`Cannot sync nested atoms! Got atom at key '{key}'`)
-		elseif isUnsafeTable(value) then
 			error(
 				`Cannot sync tables unsupported by remote events! The value has the key '{key}'.\n\n`
 					.. "This can be for the following reasons:\n"
@@ -69,7 +65,7 @@ local function validate(value: any, key: any)
 	elseif typeOfKey == "number" then
 		if key == math.huge or key == -math.huge then
 			error("Cannot sync infinity as key")
-		elseif math.floor(key) ~= key then
+		elseif key ~= math.floor(key) then
 			error("Cannot sync non-integer number as key")
 		end
 	end

src/charm/.luaurc

@@ -1,3 +0,0 @@
-{
-	"languageMode": "strict"
-}

src/charm/src/index.d.ts

@@ -10,24 +10,13 @@ type AnyMap<K, V> =
 
 declare namespace Charm {
 	/**
-	 * A primitive state container that can be read from and written to. When the
+	 * A primitive state container that can be read from or written to. When
-	 * state changes, all subscribers are notified.
+	 * the state changes, all subscribers are notified.
 	 *
-	 * @template State The type of the state.
 	 * @param state The next state or a function that produces the next state.
 	 * @returns The current state, if no arguments are provided.
 	 */
-	interface Atom<State> extends Selector<State> {
+	type Atom<State> = (state?: State | ((prev: State) => State)) => State;
-		/**
-		 * @deprecated This property is not meant to be accessed directly.
-		 */
-		readonly __nominal: unique symbol;
-		/**
-		 * @param state The next state or a function that produces the next state.
-		 * @returns The current state, if no arguments are provided.
-		 */
-		(state?: State | ((prev: State) => State)): State;
-	}
 
 	/**
 	 * A function that depends on one or more atoms and produces a value. Can be

src/charm/src/store.luau

@@ -3,6 +3,8 @@ type Atom<T> = types.Atom<T>
 type Set<T> = types.Set<T>
 type WeakMap<K, V> = types.WeakMap<K, V>
 
+local __DEV__ = _G.__DEV__
+
 local listeners: WeakMap<Atom<any>, WeakMap<() -> (), unknown>> = setmetatable({}, { __mode = "k" })
 
 local batched: Set<() -> ()> = {}
@@ -24,7 +26,7 @@ local capturing = {
 	@return The result of the callback.
 ]=]
 local function try<T, U...>(callback: (U...) -> T | any, finally: (() -> ())?, ...: U...): T
-	if _G.__DEV__ then
+	if __DEV__ then
 		local thread = coroutine.create(callback)
 		local success, result = coroutine.resume(thread, ...)
 

src/react/.luaurc

@@ -1,3 +0,0 @@
-{
-	"languageMode": "strict"
-}

src/vide/.luaurc

@@ -1,3 +0,0 @@
-{
-	"languageMode": "strict"
-}

src/vide/src/init.luau

@@ -9,10 +9,12 @@ local Vide = if script.Parent:FindFirstChild("vide")
 	@param callback The atom or selector to subscribe to.
 	@return The reactive source.
 ]=]
-local function useAtom<State>(callback: () -> State): State
+local function useAtom<State>(callback: () -> State): () -> State
 	local state = Vide.source(callback())
 	local unsubscribe = Charm.subscribe(callback, function(value)
-		task.spawn(state, value)
+		task.spawn(function()
+			state(value)
+		end)
 	end)
 
 	Vide.cleanup(unsubscribe)

tests/charm-sync/init.spec.luau

@@ -1,19 +0,0 @@
-local ReplicatedStorage = game:GetService("ReplicatedStorage")
-
-local store = require(ReplicatedStorage.DevPackages.Charm.store)
-
-return function()
-	beforeAll(function()
-		_G.__DEV__ = true
-	end)
-
-	afterAll(function()
-		_G.__DEV__ = nil
-	end)
-
-	afterEach(function()
-		table.clear(store.listeners :: any)
-		table.clear(store.capturing.stack)
-		store.capturing.index = 0
-	end)
-end

tests/charm-sync/patch.spec.luau

@@ -98,4 +98,14 @@ return function()
 		expect(applied["2"]).to.be.ok()
 		expect(applied[2]).to.never.be.ok()
 	end)
+
+	it("optionally fixes sparse arrays", function()
+		for index in next, patch.diff({ 1, 1, 1, 1, 1 }, { 2, 1, 1, 2 }) do
+			expect(index).to.be.a("string")
+		end
+
+		for index in next, patch.diff({ 1, 1, 1, 1, 1 }, { 2, 1, 1, 2 }, false) do
+			expect(index).to.be.a("number")
+		end
+	end)
 end

tests/charm-sync/server.spec.luau

@@ -49,7 +49,7 @@ return function()
 		server:_sendPatch(target)
 
 		expect(payload.data.a.b).to.never.be.ok()
-		expect(payload.data.a.c).to.equal(patch.NONE)
+		expect(patch.isNone(payload.data.a.c)).to.equal(true)
 		expect(payload.data.a.d).to.equal(1)
 
 		disconnect()

tests/init.server.luau

@@ -1,5 +1,12 @@
+--!nocheck
+
 local ReplicatedStorage = game:GetService("ReplicatedStorage")
 
 local TestEZ = require(ReplicatedStorage.DevPackages.TestEZ)
 
+-- Start Charm in debug mode
+_G.__DEV__ = true
+require(ReplicatedStorage.DevPackages.Charm)
+_G.__DEV__ = false
+
 TestEZ.TestBootstrap:run({ script.Parent }, TestEZ.Reporters.TextReporter)

tests/init.spec.luau

@@ -3,14 +3,6 @@ local ReplicatedStorage = game:GetService("ReplicatedStorage")
 local store = require(ReplicatedStorage.DevPackages.Charm.store)
 
 return function()
-	beforeAll(function()
-		_G.__DEV__ = true
-	end)
-
-	afterAll(function()
-		_G.__DEV__ = nil
-	end)
-
 	afterEach(function()
 		table.clear(store.listeners :: any)
 		table.clear(store.capturing.stack)