Binary storage for inventories (#1662)

Related to: #1642
Resolves: #1290

---------

Co-authored-by: IntegratedQuantum <43880493+IntegratedQuantum@users.noreply.github.com>
Co-authored-by: OneAvargeCoder193 <mgiakimenko@outlook.com>

src/Inventory.zig

@@ -7,6 +7,8 @@ const Item = main.items.Item;
 const ItemStack = main.items.ItemStack;
 const Tool = main.items.Tool;
 const utils = main.utils;
+const BinaryWriter = utils.BinaryWriter;
+const BinaryReader = utils.BinaryReader;
 const NeverFailingAllocator = main.heap.NeverFailingAllocator;
 const vec = main.vec;
 const Vec3d = vec.Vec3d;
@@ -314,12 +316,12 @@ pub const Sync = struct { // MARK: Sync
 			executeCommand(payload, source);
 		}
 
-		pub fn createExternallyManagedInventory(len: usize, typ: Inventory.Type, source: Source, zon: ZonElement) u32 {
+		pub fn createExternallyManagedInventory(len: usize, typ: Inventory.Type, source: Source, data: *BinaryReader) u32 {
 			mutex.lock();
 			defer mutex.unlock();
 			const inventory = ServerInventory.init(len, typ, source, .externallyManaged);
 			inventories.items[inventory.inv.id] = inventory;
-			inventory.inv.loadFromZon(zon);
+			inventory.inv.fromBytes(data);
 			return inventory.inv.id;
 		}
 
@@ -1821,7 +1823,7 @@ const SourceType = enum(u8) {
 	blockInventory = 5,
 	other = 0xff, // TODO: List every type separately here.
 };
-const Source = union(SourceType) {
+pub const Source = union(SourceType) {
 	alreadyFreed: void,
 	playerInventory: u32,
 	sharedTestingInventory: void,
@@ -1990,18 +1992,7 @@ pub fn getAmount(self: Inventory, slot: usize) u16 {
 	return self._items[slot].amount;
 }
 
-pub fn save(self: Inventory, allocator: NeverFailingAllocator) ZonElement {
-	const zonObject = ZonElement.initObject(allocator);
-	zonObject.put("capacity", self._items.len);
-	for(self._items, 0..) |stack, i| {
-		if(!stack.empty()) {
-			var buf: [1024]u8 = undefined;
-			zonObject.put(buf[0..std.fmt.printInt(&buf, i, 10, .lower, .{})], stack.store(allocator));
-		}
-	}
-	return zonObject;
-}
-
+// TODO: Remove after #480
 pub fn loadFromZon(self: Inventory, zon: ZonElement) void {
 	for(self._items, 0..) |*stack, i| {
 		stack.clear();
@@ -2019,3 +2010,33 @@ pub fn loadFromZon(self: Inventory, zon: ZonElement) void {
 		}
 	}
 }
+
+pub fn toBytes(self: Inventory, writer: *BinaryWriter) void {
+	writer.writeVarInt(u32, @intCast(self._items.len));
+	for(self._items) |stack| {
+		stack.toBytes(writer);
+	}
+}
+
+pub fn fromBytes(self: Inventory, reader: *BinaryReader) void {
+	var remainingCount = reader.readVarInt(u32) catch 0;
+	for(self._items) |*stack| {
+		if(remainingCount == 0) {
+			stack.clear();
+			continue;
+		}
+		remainingCount -= 1;
+		stack.* = ItemStack.fromBytes(reader) catch |err| {
+			std.log.err("Failed to read item stack from bytes: {s}", .{@errorName(err)});
+			stack.clear();
+			continue;
+		};
+	}
+	for(0..remainingCount) |_| {
+		var stack = ItemStack.fromBytes(reader) catch continue;
+		if(stack.item) |item| {
+			std.log.err("Lost {} of {s}", .{stack.amount, item.id()});
+		}
+		stack.deinit();
+	}
+}

src/items.zig

@@ -182,7 +182,7 @@ const MaterialProperty = enum {
 	}
 };
 
-pub const BaseItemIndex = enum(u16) {
+pub const BaseItemIndex = enum(u16) { // MARK: BaseItemIndex
 	_,
 
 	pub fn fromId(_id: []const u8) ?BaseItemIndex {
@@ -804,6 +804,10 @@ pub const Tool = struct { // MARK: Tool
 		return self.texture.?;
 	}
 
+	fn id(self: *Tool) []const u8 {
+		return self.type.id();
+	}
+
 	fn getTooltip(self: *Tool) []const u8 {
 		self.tooltip.clearRetainingCapacity();
 		self.tooltip.writer().print(
@@ -938,6 +942,14 @@ pub const Item = union(ItemType) { // MARK: Item
 		}
 	}
 
+	pub fn id(self: Item) []const u8 {
+		switch(self) {
+			inline else => |item| {
+				return item.id();
+			},
+		}
+	}
+
 	pub fn getTooltip(self: Item) []const u8 {
 		switch(self) {
 			.baseItem => |_baseItem| {

src/server/world.zig

@@ -539,6 +539,85 @@ pub const ServerWorld = struct { // MARK: ServerWorld
 		try files.writeZon(try std.fmt.allocPrint(arenaAllocator.allocator, "saves/{s}/tool_palette.zig.zon", .{path}), self.toolPalette.storeToZon(arenaAllocator));
 		try files.writeZon(try std.fmt.allocPrint(arenaAllocator.allocator, "saves/{s}/biome_palette.zig.zon", .{path}), self.biomePalette.storeToZon(arenaAllocator));
 
+		convert_player_data_to_binary: { // TODO: Remove after #480
+			std.log.debug("Migrating old player inventory format to binary.", .{});
+
+			const playerDataPath = std.fmt.allocPrint(main.stackAllocator.allocator, "saves/{s}/players", .{path}) catch unreachable;
+			defer main.stackAllocator.free(playerDataPath);
+
+			var playerDataDirectory = std.fs.cwd().openDir(playerDataPath, .{.iterate = true}) catch break :convert_player_data_to_binary;
+			defer playerDataDirectory.close();
+
+			{
+				var walker = playerDataDirectory.walk(main.stackAllocator.allocator) catch unreachable;
+				defer walker.deinit();
+
+				while(walker.next() catch |err| {
+					std.log.err("Couldn't fetch next directory entry due to an error: {s}", .{@errorName(err)});
+					break :convert_player_data_to_binary;
+				}) |entry| {
+					if(entry.kind != .file) continue;
+					if(!std.ascii.endsWithIgnoreCase(entry.basename, ".zon")) continue;
+
+					const absolutePath = std.fmt.allocPrint(main.stackAllocator.allocator, "{s}/{s}", .{playerDataPath, entry.path}) catch unreachable;
+					defer main.stackAllocator.free(absolutePath);
+
+					const playerData = files.readToZon(main.stackAllocator, absolutePath) catch |err| {
+						std.log.err("Could not read player data file '{s}'': {s}.", .{absolutePath, @errorName(err)});
+						continue;
+					};
+					defer playerData.deinit(main.stackAllocator);
+
+					std.log.debug("Migrating player data file: '{s}'", .{absolutePath});
+
+					const entryKeys: [2][]const u8 = .{
+						"playerInventory",
+						"hand",
+					};
+					for(entryKeys) |key| {
+						const zon = playerData.getChild(key);
+						switch(zon) {
+							.object => {
+								std.log.debug("Migrating inventory '{s}' '{s}'", .{key, absolutePath});
+
+								var temp: main.items.Inventory = undefined;
+								temp._items = main.stackAllocator.alloc(ItemStack, zon.get(u32, "capacity", 0));
+								defer main.stackAllocator.free(temp._items);
+
+								for(temp._items) |*stack| stack.* = ItemStack{};
+								defer for(temp._items) |*stack| stack.deinit();
+
+								temp.loadFromZon(zon);
+
+								for(temp._items, 0..) |*stack, i| {
+									std.log.debug("Item #{}: {} x {s}", .{i, stack.amount, if(stack.item) |item| item.id() else "null"});
+								}
+
+								const base64Data = savePlayerInventory(main.stackAllocator, temp);
+								const old = playerData.object.fetchPut(key, .{.stringOwned = base64Data}) catch unreachable orelse unreachable;
+								old.value.deinit(main.stackAllocator);
+							},
+							.string, .stringOwned => |field| {
+								std.log.debug("Skipping key '{s}', type is 'string', value is '{s}'", .{key, field});
+							},
+							.null => {
+								std.log.debug("Skipping key '{s}', type is 'null'", .{key});
+							},
+							else => |other| {
+								const representation = zon.toString(main.stackAllocator);
+								defer main.stackAllocator.free(representation);
+								std.log.err("Encountered unexpected type ({s}) while migrating '{s}': {s}", .{@tagName(other), absolutePath, representation});
+							},
+						}
+					}
+					files.writeZon(absolutePath, playerData) catch |err| {
+						std.log.err("Could not write player data file {s}: {s}.", .{absolutePath, @errorName(err)});
+						continue;
+					};
+				}
+			}
+		}
+
 		var gamerules = files.readToZon(arenaAllocator, try std.fmt.allocPrint(arenaAllocator.allocator, "saves/{s}/gamerules.zig.zon", .{path})) catch ZonElement.initObject(arenaAllocator);
 
 		self.defaultGamemode = std.meta.stringToEnum(main.game.Gamemode, gamerules.get([]const u8, "default_gamemode", "creative")) orelse .creative;
@@ -862,9 +941,37 @@ pub const ServerWorld = struct { // MARK: ServerWorld
 
 			main.items.Inventory.Sync.setGamemode(user, std.meta.stringToEnum(main.game.Gamemode, playerData.get([]const u8, "gamemode", @tagName(self.defaultGamemode))) orelse self.defaultGamemode);
 		}
+		user.inventory = loadPlayerInventory(main.game.Player.inventorySize, playerData.get([]const u8, "playerInventory", ""), .{.playerInventory = user.id}, path);
+		user.handInventory = loadPlayerInventory(1, playerData.get([]const u8, "hand", ""), .{.hand = user.id}, path);
+	}
 
-		user.inventory = main.items.Inventory.Sync.ServerSide.createExternallyManagedInventory(main.game.Player.inventorySize, .normal, .{.playerInventory = user.id}, playerData.getChild("playerInventory"));
-		user.handInventory = main.items.Inventory.Sync.ServerSide.createExternallyManagedInventory(1, .normal, .{.hand = user.id}, playerData.getChild("hand"));
+	fn loadPlayerInventory(size: usize, base64EncodedData: []const u8, source: main.items.Inventory.Source, playerDataFilePath: []const u8) u32 {
+		const decodedSize = std.base64.url_safe.Decoder.calcSizeForSlice(base64EncodedData) catch |err| blk: {
+			std.log.err("Encountered incorrectly encoded inventory data ({s}) while loading data from file '{s}': '{s}'", .{@errorName(err), playerDataFilePath, base64EncodedData});
+			break :blk 0;
+		};
+
+		const bytes: []u8 = main.stackAllocator.alloc(u8, decodedSize);
+		defer main.stackAllocator.free(bytes);
+
+		var readerInput: []const u8 = bytes;
+
+		std.base64.url_safe.Decoder.decode(bytes, base64EncodedData) catch |err| {
+			std.log.err("Encountered incorrectly encoded inventory data ({s}) while loading data from file '{s}': '{s}'", .{@errorName(err), playerDataFilePath, base64EncodedData});
+			readerInput = "";
+		};
+		var reader: main.utils.BinaryReader = .init(readerInput);
+		return main.items.Inventory.Sync.ServerSide.createExternallyManagedInventory(size, .normal, source, &reader);
+	}
+
+	fn savePlayerInventory(allocator: NeverFailingAllocator, inv: main.items.Inventory) []const u8 {
+		var writer = main.utils.BinaryWriter.init(main.stackAllocator);
+		defer writer.deinit();
+
+		inv.toBytes(&writer);
+
+		const destination: []u8 = allocator.alloc(u8, std.base64.url_safe.Encoder.calcSize(writer.data.items.len));
+		return std.base64.url_safe.Encoder.encode(destination, writer.data.items);
 	}
 
 	pub fn savePlayer(self: *ServerWorld, user: *User) !void {
@@ -892,11 +999,11 @@ pub const ServerWorld = struct { // MARK: ServerWorld
 			main.items.Inventory.Sync.ServerSide.mutex.lock();
 			defer main.items.Inventory.Sync.ServerSide.mutex.unlock();
 			if(main.items.Inventory.Sync.ServerSide.getInventoryFromSource(.{.playerInventory = user.id})) |inv| {
-				playerZon.put("playerInventory", inv.save(main.stackAllocator));
+				playerZon.put("playerInventory", ZonElement{.stringOwned = savePlayerInventory(main.stackAllocator, inv)});
 			} else @panic("The player inventory wasn't found. Cannot save player data.");
 
 			if(main.items.Inventory.Sync.ServerSide.getInventoryFromSource(.{.hand = user.id})) |inv| {
-				playerZon.put("hand", inv.save(main.stackAllocator));
+				playerZon.put("hand", ZonElement{.stringOwned = savePlayerInventory(main.stackAllocator, inv)});
 			} else @panic("The player hand inventory wasn't found. Cannot save player data.");
 		}