Implement some basic lossy compression for entity and item network data

progress towards #1327

src/entity.zig

@@ -18,6 +18,13 @@ const NeverFailingAllocator = main.heap.NeverFailingAllocator;
 
 const BinaryReader = main.utils.BinaryReader;
 
+pub const EntityNetworkData = struct {
+	id: u32,
+	pos: Vec3d,
+	vel: Vec3d,
+	rot: Vec3f,
+};
+
 pub const ClientEntity = struct {
 	interpolatedValues: utils.GenericInterpolation(6) = undefined,
 	_interpolationPos: [6]f64 = undefined,
@@ -228,31 +235,30 @@ pub const ClientEntityManager = struct {
 		}
 	}
 
-	pub fn serverUpdate(time: i16, reader: *BinaryReader) !void {
+	pub fn serverUpdate(time: i16, entityData: []EntityNetworkData) void {
 		mutex.lock();
 		defer mutex.unlock();
 		timeDifference.addDataPoint(time);
 
-		while(reader.remaining.len != 0) {
+		for(entityData) |data| {
-			const id = try reader.readInt(u32);
 			const pos = [_]f64{
-				try reader.readFloat(f64),
+				data.pos[0],
-				try reader.readFloat(f64),
+				data.pos[1],
-				try reader.readFloat(f64),
+				data.pos[2],
-				@floatCast(try reader.readFloat(f32)),
+				@floatCast(data.rot[0]),
-				@floatCast(try reader.readFloat(f32)),
+				@floatCast(data.rot[1]),
-				@floatCast(try reader.readFloat(f32)),
+				@floatCast(data.rot[2]),
 			};
 			const vel = [_]f64{
-				try reader.readFloat(f64),
+				data.vel[0],
-				try reader.readFloat(f64),
+				data.vel[1],
-				try reader.readFloat(f64),
+				data.vel[2],
 				0,
 				0,
 				0,
 			};
 			for(entities.items()) |*ent| {
-				if(ent.id == id) {
+				if(ent.id == data.id) {
 					ent.updatePosition(&pos, &vel, time);
 					break;
 				}

src/itemdrop.zig

@@ -35,6 +35,12 @@ const ItemDrop = struct { // MARK: ItemDrop
 	reverseIndex: u16,
 };
 
+pub const ItemDropNetworkData = struct {
+	index: u16,
+	pos: Vec3d,
+	vel: Vec3d,
+};
+
 pub const ItemDropManager = struct { // MARK: ItemDropManager
 	/// Half the side length of all item entities hitboxes as a cube.
 	pub const radius: f64 = 0.1;
@@ -116,14 +122,16 @@ pub const ItemDropManager = struct { // MARK: ItemDropManager
 		}
 	}
 
-	pub fn getPositionAndVelocityData(self: *ItemDropManager, allocator: NeverFailingAllocator) []u8 {
+	pub fn getPositionAndVelocityData(self: *ItemDropManager, allocator: NeverFailingAllocator) []ItemDropNetworkData {
-		var writer = utils.BinaryWriter.initCapacity(allocator, self.size*50);
+		const result = allocator.alloc(ItemDropNetworkData, self.size);
-		for(self.indices[0..self.size]) |i| {
+		for(self.indices[0..self.size], result) |i, *res| {
-			writer.writeInt(u16, i);
+			res.* = .{
-			writer.writeVec(Vec3d, self.list.items(.pos)[i]);
+				.index = i,
-			writer.writeVec(Vec3d, self.list.items(.vel)[i]);
+				.pos = self.list.items(.pos)[i],
+				.vel = self.list.items(.vel)[i],
+			};
 		}
-		return writer.data.toOwnedSlice();
+		return result;
 	}
 
 	pub fn getInitialList(self: *ItemDropManager, allocator: NeverFailingAllocator) ZonElement {
@@ -458,14 +466,13 @@ pub const ClientItemDropManager = struct { // MARK: ClientItemDropManager
 		self.super.deinit();
 	}
 
-	pub fn readPosition(self: *ClientItemDropManager, reader: *BinaryReader, time: i16) !void {
+	pub fn readPosition(self: *ClientItemDropManager, time: i16, itemData: []ItemDropNetworkData) void {
 		self.timeDifference.addDataPoint(time);
 		var pos: [ItemDropManager.maxCapacity]Vec3d = undefined;
 		var vel: [ItemDropManager.maxCapacity]Vec3d = undefined;
-		while(reader.remaining.len != 0) {
+		for(itemData) |data| {
-			const i = try reader.readInt(u16);
+			pos[data.index] = data.pos;
-			pos[i] = try reader.readVec(Vec3d);
+			vel[data.index] = data.vel;
-			vel[i] = try reader.readVec(Vec3d);
 		}
 		mutex.lock();
 		defer mutex.unlock();

src/network.zig

@@ -850,35 +850,93 @@ pub const Protocols = struct {
 		pub const asynchronous = false;
 		const type_entity: u8 = 0;
 		const type_item: u8 = 1;
+		const Type = enum(u8) {
+			noVelocityEntity = 0,
+			f16VelocityEntity = 1,
+			f32VelocityEntity = 2,
+			noVelocityItem = 3,
+			f16VelocityItem = 4,
+			f32VelocityItem = 5,
+		};
 		fn receive(conn: *Connection, reader: *utils.BinaryReader) !void {
+			if(conn.isServerSide()) return error.InvalidSide;
 			if(conn.manager.world) |world| {
-				const typ = try reader.readInt(u8);
 				const time = try reader.readInt(i16);
-				if(typ == type_entity) {
+				const playerPos = try reader.readVec(Vec3d);
-					try main.entity.ClientEntityManager.serverUpdate(time, reader);
+				var entityData: main.List(main.entity.EntityNetworkData) = .init(main.stackAllocator);
-				} else if(typ == type_item) {
+				defer entityData.deinit();
-					try world.itemDrops.readPosition(reader, time);
+				var itemData: main.List(main.itemdrop.ItemDropNetworkData) = .init(main.stackAllocator);
+				defer itemData.deinit();
+				while(reader.remaining.len != 0) {
+					const typ = try reader.readEnum(Type);
+					switch(typ) {
+						.noVelocityEntity, .f16VelocityEntity, .f32VelocityEntity => {
+							entityData.append(.{
+								.vel = switch(typ) {
+									.noVelocityEntity => @splat(0),
+									.f16VelocityEntity => @floatCast(try reader.readVec(@Vector(3, f16))),
+									.f32VelocityEntity => @floatCast(try reader.readVec(@Vector(3, f32))),
+									else => unreachable,
+								},
+								.id = try reader.readInt(u32),
+								.pos = playerPos + try reader.readVec(Vec3f),
+								.rot = try reader.readVec(Vec3f),
+							});
+						},
+						.noVelocityItem, .f16VelocityItem, .f32VelocityItem => {
+							itemData.append(.{
+								.vel = switch(typ) {
+									.noVelocityItem => @splat(0),
+									.f16VelocityItem => @floatCast(try reader.readVec(@Vector(3, f16))),
+									.f32VelocityItem => @floatCast(try reader.readVec(Vec3f)),
+									else => unreachable,
+								},
+								.index = try reader.readInt(u16),
+								.pos = playerPos + try reader.readVec(Vec3f),
+							});
+						},
+					}
 				}
+				main.entity.ClientEntityManager.serverUpdate(time, entityData.items);
+				world.itemDrops.readPosition(time, itemData.items);
 			}
 		}
-		pub fn send(conn: *Connection, entityData: []const u8, itemData: []const u8) void {
+		pub fn send(conn: *Connection, playerPos: Vec3d, entityData: []main.entity.EntityNetworkData, itemData: []main.itemdrop.ItemDropNetworkData) void {
-			if(entityData.len != 0) {
+			var writer = utils.BinaryWriter.init(main.stackAllocator);
-				var writer = utils.BinaryWriter.initCapacity(main.stackAllocator, entityData.len + 3);
+			defer writer.deinit();
-				defer writer.deinit();
-				writer.writeInt(u8, type_entity);
-				writer.writeInt(i16, @truncate(std.time.milliTimestamp()));
-				writer.writeSlice(entityData);
-				conn.send(.lossy, id, writer.data.items);
-			}
 
-			if(itemData.len != 0) {
+			writer.writeInt(i16, @truncate(std.time.milliTimestamp()));
-				var writer = utils.BinaryWriter.initCapacity(main.stackAllocator, itemData.len + 3);
+			writer.writeVec(Vec3d, playerPos);
-				defer writer.deinit();
+			for(entityData) |data| {
-				writer.writeInt(u8, type_item);
+				const velocityMagnitudeSqr = vec.lengthSquare(data.vel);
-				writer.writeInt(i16, @truncate(std.time.milliTimestamp()));
+				if(velocityMagnitudeSqr < 1e-6*1e-6) {
-				writer.writeSlice(itemData);
+					writer.writeEnum(Type, .noVelocityEntity);
-				conn.send(.lossy, id, writer.data.items);
+				} else if(velocityMagnitudeSqr > 1000*1000) {
+					writer.writeEnum(Type, .f32VelocityEntity);
+					writer.writeVec(Vec3f, @floatCast(data.vel));
+				} else {
+					writer.writeEnum(Type, .f16VelocityEntity);
+					writer.writeVec(@Vector(3, f16), @floatCast(data.vel));
+				}
+				writer.writeInt(u32, data.id);
+				writer.writeVec(Vec3f, @floatCast(data.pos - playerPos));
+				writer.writeVec(Vec3f, data.rot);
 			}
+			for(itemData) |data| {
+				const velocityMagnitudeSqr = vec.lengthSquare(data.vel);
+				if(velocityMagnitudeSqr < 1e-6*1e-6) {
+					writer.writeEnum(Type, .noVelocityItem);
+				} else if(velocityMagnitudeSqr > 1000*1000) {
+					writer.writeEnum(Type, .f32VelocityItem);
+					writer.writeVec(Vec3f, @floatCast(data.vel));
+				} else {
+					writer.writeEnum(Type, .f16VelocityItem);
+					writer.writeVec(@Vector(3, f16), @floatCast(data.vel));
+				}
+				writer.writeInt(u16, data.index);
+				writer.writeVec(Vec3f, @floatCast(data.pos - playerPos));
+			}
+			conn.send(.lossy, id, writer.data.items);
 		}
 	};
 	pub const blockUpdate = struct {

src/server/server.zig

@@ -387,21 +387,23 @@ fn update() void { // MARK: update()
 	}
 
 	// Send the entity data:
-	var writer = BinaryWriter.initCapacity(main.stackAllocator, (4 + 24 + 12 + 24)*userList.len);
-	defer writer.deinit();
-
 	const itemData = world.?.itemDropManager.getPositionAndVelocityData(main.stackAllocator);
 	defer main.stackAllocator.free(itemData);
 
+	var entityData: main.List(main.entity.EntityNetworkData) = .init(main.stackAllocator);
+	defer entityData.deinit();
+
 	for(userList) |user| {
 		const id = user.id; // TODO
-		writer.writeInt(u32, id);
+		entityData.append(.{
-		writer.writeVec(Vec3d, user.player.pos);
+			.id = id,
-		writer.writeVec(Vec3f, user.player.rot);
+			.pos = user.player.pos,
-		writer.writeVec(Vec3d, user.player.vel);
+			.vel = user.player.vel,
+			.rot = user.player.rot,
+		});
 	}
 	for(userList) |user| {
-		main.network.Protocols.entityPosition.send(user.conn, writer.data.items, itemData);
+		main.network.Protocols.entityPosition.send(user.conn, user.player.pos, entityData.items, itemData);
 	}
 
 	while(userDeinitList.dequeue()) |user| {