From 758b9ec21b1e17aaa6c93bfabd46c23070ba59eb Mon Sep 17 00:00:00 2001 From: IntegratedQuantum Date: Sun, 12 May 2024 15:35:37 +0200 Subject: [PATCH] Load and store chunks (it is a bit fragile still) A big step towards #80, still needs some work to ensure all chunks are stored, including updating lod chunks. --- src/chunk.zig | 96 +++++++++++++++++++----------------------- src/files.zig | 4 ++ src/network.zig | 8 +++- src/renderer.zig | 19 +++++---- src/server/storage.zig | 90 ++++++++++++++++++++++++++++++++++----- src/server/world.zig | 30 ++++++++++++- 6 files changed, 173 insertions(+), 74 deletions(-) diff --git a/src/chunk.zig b/src/chunk.zig index b6be52ae..e76d51c1 100644 --- a/src/chunk.zig +++ b/src/chunk.zig @@ -174,8 +174,6 @@ pub const Chunk = struct { data: main.utils.PaletteCompressedRegion(Block, chunkVolume) = undefined, wasChanged: bool = false, - /// When a chunk is cleaned, it won't be saved by the ChunkManager anymore, so following changes need to be saved directly. - wasCleaned: bool = false, generated: bool = false, width: u31, @@ -210,35 +208,6 @@ pub const Chunk = struct { memoryPoolMutex.unlock(); } - pub fn setChanged(self: *Chunk) void { - self.wasChanged = true; - { - self.mutex.lock(); - if(self.wasCleaned) { - self.save(); - } - self.mutex.unlock(); - } - } - - pub fn clean(self: *Chunk) void { - { - self.mutex.lock(); - self.wasCleaned = true; - self.save(); - self.mutex.unlock(); - } - } - - pub fn unclean(self: *Chunk) void { - { - self.mutex.lock(); - self.wasCleaned = false; - self.save(); - self.mutex.unlock(); - } - } - /// Checks if the given relative coordinates lie within the bounds of this chunk. pub fn liesInChunk(self: *const Chunk, x: i32, y: i32, z: i32) bool { return x >= 0 and x < self.width @@ -278,6 +247,17 @@ pub const Chunk = struct { self.data.setValue(index, newBlock); } + /// Updates a block if it is inside this chunk. + /// Does not do any bound checks. They are expected to be done with the `liesInChunk` function. + pub fn updateBlockAndSetChanged(self: *Chunk, _x: i32, _y: i32, _z: i32, newBlock: Block) void { + const x = _x >> self.voxelSizeShift; + const y = _y >> self.voxelSizeShift; + const z = _z >> self.voxelSizeShift; + const index = getIndex(x, y, z); + self.data.setValue(index, newBlock); + self.wasChanged = true; + } + /// Updates a block if it is inside this chunk. Should be used in generation to prevent accidently storing these as changes. /// Does not do any bound checks. They are expected to be done with the `liesInChunk` function. pub fn updateBlockInGeneration(self: *Chunk, _x: i32, _y: i32, _z: i32, newBlock: Block) void { @@ -299,32 +279,32 @@ pub const Chunk = struct { } pub fn updateFromLowerResolution(self: *Chunk, other: *const Chunk) void { - const xOffset = if(other.wx != self.wx) chunkSize/2 else 0; // Offsets of the lower resolution chunk in this chunk. - const yOffset = if(other.wy != self.wy) chunkSize/2 else 0; - const zOffset = if(other.wz != self.wz) chunkSize/2 else 0; + const xOffset = if(other.pos.wx != self.pos.wx) chunkSize/2 else 0; // Offsets of the lower resolution chunk in this chunk. + const yOffset = if(other.pos.wy != self.pos.wy) chunkSize/2 else 0; + const zOffset = if(other.pos.wz != self.pos.wz) chunkSize/2 else 0; - var x: i32 = 0; + var x: u31 = 0; while(x < chunkSize/2): (x += 1) { - var y: i32 = 0; + var y: u31 = 0; while(y < chunkSize/2): (y += 1) { - var z: i32 = 0; + var z: u31 = 0; while(z < chunkSize/2): (z += 1) { // Count the neighbors for each subblock. An transparent block counts 5. A chunk border(unknown block) only counts 1. - var neighborCount: [8]u32 = undefined; + var neighborCount: [8]u31 = undefined; var octantBlocks: [8]Block = undefined; - var maxCount: u32 = 0; - var dx: i32 = 0; + var maxCount: i32 = 0; + var dx: u31 = 0; while(dx <= 1): (dx += 1) { - var dy: i32 = 0; + var dy: u31 = 0; while(dy <= 1): (dy += 1) { - var dz: i32 = 0; + var dz: u31 = 0; while(dz <= 1): (dz += 1) { const index = getIndex(x*2 + dx, y*2 + dy, z*2 + dz); const i = dx*4 + dz*2 + dy; octantBlocks[i] = other.data.getValue(index); - if(octantBlocks[i] == 0) continue; // I don't care about air blocks. + if(octantBlocks[i].typ == 0) continue; // I don't care about air blocks. - var count: u32 = 0; + var count: u31 = 0; for(Neighbors.iterable) |n| { const nx = x*2 + dx + Neighbors.relX[n]; const ny = y*2 + dy + Neighbors.relY[n]; @@ -345,7 +325,7 @@ pub const Chunk = struct { } // Uses a specific permutation here that keeps high resolution patterns in lower resolution. const permutationStart = (x & 1)*4 + (z & 1)*2 + (y & 1); - const block = Block{.typ = 0, .data = 0}; + var block = Block{.typ = 0, .data = 0}; for(0..8) |i| { const appliedPermutation = permutationStart ^ i; if(neighborCount[appliedPermutation] >= maxCount - 1) { // Avoid pattern breaks at chunk borders. @@ -358,24 +338,36 @@ pub const Chunk = struct { } } } - - self.setChanged(); + + self.wasChanged = true; } pub fn save(self: *Chunk, world: *main.server.ServerWorld) void { self.mutex.lock(); defer self.mutex.unlock(); if(self.wasChanged) { - // TODO: Actually store the chunk + const regionSize = self.pos.voxelSize*chunkSize*main.server.storage.RegionFile.regionSize; + const regionMask: i32 = regionSize - 1; + const region = main.server.storage.loadRegionFileAndIncreaseRefCount(self.pos.wx & ~regionMask, self.pos.wy & ~regionMask, self.pos.wz & ~regionMask, self.pos.voxelSize); + defer region.decreaseRefCount(); + const data = main.server.storage.ChunkCompression.compressChunk(main.stackAllocator, self); + defer main.stackAllocator.free(data); + region.storeChunk( + data, + @as(usize, @intCast(self.pos.wx -% region.pos.wx))/self.pos.voxelSize/chunkSize, + @as(usize, @intCast(self.pos.wy -% region.pos.wy))/self.pos.voxelSize/chunkSize, + @as(usize, @intCast(self.pos.wz -% region.pos.wz))/self.pos.voxelSize/chunkSize, + ); + self.wasChanged = false; // Update the next lod chunk: if(self.pos.voxelSize != 1 << settings.highestLOD) { var pos = self.pos; - pos.wx &= ~pos.voxelSize; - pos.wy &= ~pos.voxelSize; - pos.wz &= ~pos.voxelSize; + pos.wx &= ~(pos.voxelSize*chunkSize); + pos.wy &= ~(pos.voxelSize*chunkSize); + pos.wz &= ~(pos.voxelSize*chunkSize); pos.voxelSize *= 2; - const nextHigherLod = world.chunkManager.getOrGenerateChunk(pos); + const nextHigherLod = world.getOrGenerateChunk(pos); nextHigherLod.updateFromLowerResolution(self); } } diff --git a/src/files.zig b/src/files.zig index a0c42571..9112ad78 100644 --- a/src/files.zig +++ b/src/files.zig @@ -26,6 +26,10 @@ pub fn openDir(path: []const u8) !Dir { }; } +pub fn makeDir(path: []const u8) !void { + try std.fs.cwd().makePath(path); +} + fn cwd() Dir { return Dir { .dir = std.fs.cwd(), diff --git a/src/network.zig b/src/network.zig index f6f385a8..ae0ade4f 100644 --- a/src/network.zig +++ b/src/network.zig @@ -725,7 +725,9 @@ pub const Protocols = struct { renderer.mesh_storage.updateChunkMesh(ch); } fn sendChunkOverTheNetwork(conn: *Connection, ch: *chunk.Chunk) void { + ch.mutex.lock(); const data = main.server.storage.ChunkCompression.compressChunk(main.stackAllocator, ch); + ch.mutex.unlock(); defer main.stackAllocator.free(data); conn.sendImportant(id, data); } @@ -816,7 +818,11 @@ pub const Protocols = struct { const z = std.mem.readInt(i32, data[8..12], .big); const newBlock = Block.fromInt(std.mem.readInt(u32, data[12..16], .big)); if(conn.user != null) { - // TODO: Handle block update from the client. + // TODO: Store changes in batches to reduce cost of singe block updates. + const mask = ~@as(i32, chunk.chunkMask); + const ch = main.server.world.?.getOrGenerateChunk(.{.wx = x & mask, .wy = y & mask, .wz = z & mask, .voxelSize = 1}); + ch.updateBlockAndSetChanged(x & chunk.chunkMask, y & chunk.chunkMask, z & chunk.chunkMask, newBlock); + ch.save(main.server.world.?); } else { renderer.mesh_storage.updateBlock(x, y, z, newBlock); } diff --git a/src/renderer.zig b/src/renderer.zig index 25600209..4a723586 100644 --- a/src/renderer.zig +++ b/src/renderer.zig @@ -748,8 +748,7 @@ pub const MeshSelection = struct { if(itemBlock == block.typ) { const relPos: Vec3f = @floatCast(lastPos - @as(Vec3d, @floatFromInt(selectedPos))); if(rotationMode.generateData(main.game.world.?, selectedPos, relPos, lastDir, neighborDir, &block, false)) { - // TODO: world.updateBlock(bi.x, bi.y, bi.z, block.data); (→ Sending it over the network) - mesh_storage.updateBlock(selectedPos[0], selectedPos[1], selectedPos[2], block); + updateBlockAndSendUpdate(selectedPos[0], selectedPos[1], selectedPos[2], block); _ = inventoryStack.add(item, @as(i32, -1)); return; } @@ -761,8 +760,7 @@ pub const MeshSelection = struct { block = mesh_storage.getBlock(neighborPos[0], neighborPos[1], neighborPos[2]) orelse return; if(block.typ == itemBlock) { if(rotationMode.generateData(main.game.world.?, neighborPos, relPos, lastDir, neighborDir, &block, false)) { - // TODO: world.updateBlock(bi.x, bi.y, bi.z, block.data); (→ Sending it over the network) - mesh_storage.updateBlock(neighborPos[0], neighborPos[1], neighborPos[2], block); + updateBlockAndSendUpdate(neighborPos[0], neighborPos[1], neighborPos[2], block); _ = inventoryStack.add(item, @as(i32, -1)); return; } @@ -772,8 +770,7 @@ pub const MeshSelection = struct { block.typ = itemBlock; block.data = 0; if(rotationMode.generateData(main.game.world.?, neighborPos, relPos, lastDir, neighborDir, &block, true)) { - // TODO: world.updateBlock(bi.x, bi.y, bi.z, block.data); (→ Sending it over the network) - mesh_storage.updateBlock(neighborPos[0], neighborPos[1], neighborPos[2], block); + updateBlockAndSendUpdate(neighborPos[0], neighborPos[1], neighborPos[2], block); _ = inventoryStack.add(item, @as(i32, -1)); return; } @@ -798,8 +795,7 @@ pub const MeshSelection = struct { if(baseItem.leftClickUse) |leftClick| { const relPos: Vec3f = @floatCast(lastPos - @as(Vec3d, @floatFromInt(selectedPos))); if(leftClick(main.game.world.?, selectedPos, relPos, lastDir, &block)) { - // TODO: world.updateBlock(bi.x, bi.y, bi.z, block.data); (→ Sending it over the network) - mesh_storage.updateBlock(selectedPos[0], selectedPos[1], selectedPos[2], block); + updateBlockAndSendUpdate(selectedPos[0], selectedPos[1], selectedPos[2], block); } return; } @@ -807,10 +803,15 @@ pub const MeshSelection = struct { else => {}, } } - mesh_storage.updateBlock(selectedPos[0], selectedPos[1], selectedPos[2], .{.typ = 0, .data = 0}); + updateBlockAndSendUpdate(selectedPos[0], selectedPos[1], selectedPos[2], .{.typ = 0, .data = 0}); } } + fn updateBlockAndSendUpdate(x: i32, y: i32, z: i32, newBlock: blocks.Block) void { + main.network.Protocols.blockUpdate.send(main.game.world.?.conn, x, y, z, newBlock); + mesh_storage.updateBlock(x, y, z, newBlock); + } + pub fn drawCube(projectionMatrix: Mat4f, viewMatrix: Mat4f, relativePositionToPlayer: Vec3d, min: Vec3f, max: Vec3f) void { shader.bind(); diff --git a/src/server/storage.zig b/src/server/storage.zig index 60874707..a4779fc6 100644 --- a/src/server/storage.zig +++ b/src/server/storage.zig @@ -1,13 +1,15 @@ const std = @import("std"); +const Atomic = std.atomic.Value; const main = @import("root"); const chunk = main.chunk; +const server = @import("server.zig"); pub const RegionFile = struct { const version = 0; - const regionShift = 2; - const regionSize = 1 << regionShift; - const regionVolume = 1 << 3*regionShift; + pub const regionShift = 2; + pub const regionSize = 1 << regionShift; + pub const regionVolume = 1 << 3*regionShift; const headerSize = 8 + regionSize*regionSize*regionSize*@sizeOf(u32); @@ -15,6 +17,8 @@ pub const RegionFile = struct { pos: chunk.ChunkPosition, mutex: std.Thread.Mutex = .{}, modified: bool = false, + refCount: Atomic(u16) = Atomic(u16).init(1), + saveFolder: []const u8, fn getIndex(x: usize, y: usize, z: usize) usize { std.debug.assert(x < regionSize and y < regionSize and z < regionSize); @@ -28,6 +32,7 @@ pub const RegionFile = struct { const self = main.globalAllocator.create(RegionFile); self.* = .{ .pos = pos, + .saveFolder = main.globalAllocator.dupe(u8, saveFolder), }; const path = std.fmt.allocPrint(main.stackAllocator.allocator, "{s}/{}/{}/{}/{}.region", .{saveFolder, pos.voxelSize, pos.wx, pos.wy, pos.wz}) catch unreachable; @@ -49,7 +54,7 @@ pub const RegionFile = struct { std.log.err("Region file {s} has incorrect version {}. Requires version {}.", .{path, fileVersion, version}); return self; } - const sizes: [regionVolume] u32 = undefined; + var sizes: [regionVolume] u32 = undefined; var totalSize: usize = 0; for(0..regionVolume) |j| { const size = std.mem.readInt(u32, data[i..][0..4], .big); @@ -70,17 +75,36 @@ pub const RegionFile = struct { } } std.debug.assert(i == data.len); + return self; } pub fn deinit(self: *RegionFile) void { + std.debug.assert(self.refCount.raw == 0); std.debug.assert(!self.modified); for(self.chunks) |ch| { main.globalAllocator.free(ch); } + main.globalAllocator.free(self.saveFolder); main.globalAllocator.destroy(self); } - pub fn store(self: *RegionFile, saveFolder: []const u8) void { + pub fn increaseRefCount(self: *RegionFile) void { + const prevVal = self.refCount.fetchAdd(1, .monotonic); + std.debug.assert(prevVal != 0); + } + + pub fn decreaseRefCount(self: *RegionFile) void { + const prevVal = self.refCount.fetchSub(1, .monotonic); + std.debug.assert(prevVal != 0); + if(prevVal == 1) { + if(self.modified) { + self.store(); + } + self.deinit(); + } + } + + pub fn store(self: *RegionFile) void { self.mutex.lock(); defer self.mutex.unlock(); self.modified = false; @@ -95,6 +119,7 @@ pub const RegionFile = struct { } const data = main.stackAllocator.alloc(u8, totalSize + headerSize); + defer main.stackAllocator.free(data); var i: usize = 0; std.mem.writeInt(u32, data[i..][0..4], version, .big); i += 4; @@ -113,8 +138,14 @@ pub const RegionFile = struct { } std.debug.assert(i == data.len); - const path = std.fmt.allocPrint(main.stackAllocator.allocator, "{s}/{}/{}/{}/{}.region", .{saveFolder, self.pos.voxelSize, self.pos.wx, self.pos.wy, self.pos.wz}) catch unreachable; + const path = std.fmt.allocPrint(main.stackAllocator.allocator, "{s}/{}/{}/{}/{}.region", .{self.saveFolder, self.pos.voxelSize, self.pos.wx, self.pos.wy, self.pos.wz}) catch unreachable; defer main.stackAllocator.free(path); + const folder = std.fmt.allocPrint(main.stackAllocator.allocator, "{s}/{}/{}/{}", .{self.saveFolder, self.pos.voxelSize, self.pos.wx, self.pos.wy}) catch unreachable; + defer main.stackAllocator.free(folder); + + main.files.makeDir(folder) catch |err| { + std.log.err("Error while writing to file {s}: {s}", .{path, @errorName(err)}); + }; main.files.write(path, data) catch |err| { std.log.err("Error while writing to file {s}: {s}", .{path, @errorName(err)}); @@ -124,20 +155,59 @@ pub const RegionFile = struct { pub fn storeChunk(self: *RegionFile, ch: []const u8, relX: usize, relY: usize, relZ: usize) void { self.mutex.lock(); defer self.mutex.unlock(); + self.modified = true; const index = getIndex(relX, relY, relZ); self.chunks[index] = main.globalAllocator.realloc(self.chunks[index], ch.len); @memcpy(self.chunks[index], ch); } + + pub fn getChunk(self: *RegionFile, allocator: main.utils.NeverFailingAllocator, relX: usize, relY: usize, relZ: usize) ?[]const u8 { + self.mutex.lock(); + defer self.mutex.unlock(); + const index = getIndex(relX, relY, relZ); + const ch = self.chunks[index]; + if(ch.len == 0) return null; + return allocator.dupe(u8, ch); + } }; + +const cacheSize = 1 << 8; // Must be a power of 2! +const cacheMask = cacheSize - 1; +const associativity = 8; +var cache: main.utils.Cache(RegionFile, cacheSize, associativity, RegionFile.decreaseRefCount) = .{}; + +fn cacheInit(pos: chunk.ChunkPosition) *RegionFile { + const path: []const u8 = std.fmt.allocPrint(main.stackAllocator.allocator, "saves/{s}/chunks", .{server.world.?.name}) catch unreachable; + defer main.stackAllocator.free(path); + return RegionFile.init(pos, path); +} + +pub fn init() void { +} + +pub fn deinit() void { + cache.clear(); +} + +pub fn loadRegionFileAndIncreaseRefCount(wx: i32, wy: i32, wz: i32, voxelSize: u31) *RegionFile { + const compare = chunk.ChunkPosition { + .wx = wx & ~@as(i32, RegionFile.regionSize*voxelSize - 1), + .wy = wy & ~@as(i32, RegionFile.regionSize*voxelSize - 1), + .wz = wz & ~@as(i32, RegionFile.regionSize*voxelSize - 1), + .voxelSize = voxelSize, + }; + const result = cache.findOrCreate(compare, cacheInit, RegionFile.increaseRefCount); + return result; +} + pub const ChunkCompression = struct { const CompressionAlgo = enum(u32) { - deflate = 0, - _, + deflate = 0, // TODO: Investigate if palette compression (or palette compression with huffman coding) is more efficient. + _, // TODO: Add more algorithms for specific cases like uniform chunks. }; pub fn compressChunk(allocator: main.utils.NeverFailingAllocator, ch: *chunk.Chunk) []const u8 { - ch.mutex.lock(); - defer ch.mutex.unlock(); + main.utils.assertLocked(&ch.mutex); var uncompressedData: [chunk.chunkVolume*@sizeOf(u32)]u8 = undefined; for(0..chunk.chunkVolume) |i| { std.mem.writeInt(u32, uncompressedData[4*i..][0..4], ch.data.getValue(i).toInt(), .big); diff --git a/src/server/world.zig b/src/server/world.zig index 2d65cc33..5405feeb 100644 --- a/src/server/world.zig +++ b/src/server/world.zig @@ -153,6 +153,7 @@ const ChunkManager = struct { .terrainGenerationProfile = try server.terrain.TerrainGenerationProfile.init(settings, world.seed), }; server.terrain.init(self.terrainGenerationProfile); + storage.init(); return self; } @@ -162,6 +163,7 @@ const ChunkManager = struct { server.terrain.deinit(); main.assets.unloadAssets(); self.terrainGenerationProfile.deinit(); + storage.deinit(); } pub fn queueLightMap(self: ChunkManager, pos: terrain.SurfaceMap.MapFragmentPosition, source: ?*User) void { @@ -188,9 +190,24 @@ const ChunkManager = struct { } fn chunkInitFunctionForCache(pos: ChunkPosition) *Chunk { + const regionSize = pos.voxelSize*chunk.chunkSize*storage.RegionFile.regionSize; + const regionMask: i32 = regionSize - 1; + const region = storage.loadRegionFileAndIncreaseRefCount(pos.wx & ~regionMask, pos.wy & ~regionMask, pos.wz & ~regionMask, pos.voxelSize); + defer region.decreaseRefCount(); + if(region.getChunk( + main.stackAllocator, + @as(usize, @intCast(pos.wx -% region.pos.wx))/pos.voxelSize/chunk.chunkSize, + @as(usize, @intCast(pos.wy -% region.pos.wy))/pos.voxelSize/chunk.chunkSize, + @as(usize, @intCast(pos.wz -% region.pos.wz))/pos.voxelSize/chunk.chunkSize, + )) |ch| blk: { // Load chunk from file: + defer main.stackAllocator.free(ch); + return storage.ChunkCompression.decompressChunk(ch) catch { + std.log.err("Storage for chunk {} in region file at {} is corrupted", .{pos, region.pos}); + break :blk; + }; + } const ch = Chunk.init(pos); ch.generated = true; - // TODO: Try loading chunk from file const caveMap = terrain.CaveMap.CaveMapView.init(ch); defer caveMap.deinit(); const biomeMap = terrain.CaveBiomeMap.CaveBiomeMapView.init(ch); @@ -207,10 +224,14 @@ const ChunkManager = struct { } /// Generates a normal chunk at a given location, or if possible gets it from the cache. pub fn getOrGenerateChunk(pos: ChunkPosition) *Chunk { // TODO: This is not thread safe! The chunk could get removed from the cache while in use. Reference counting should probably be used here. + const mask = pos.voxelSize*chunk.chunkSize - 1; + std.debug.assert(pos.wx & mask == 0 and pos.wy & mask == 0 and pos.wz & mask == 0); return chunkCache.findOrCreate(pos, chunkInitFunctionForCache, null); } pub fn getChunkFromCache(pos: ChunkPosition) ?*Chunk { + const mask = pos.voxelSize*chunk.chunkSize - 1; + std.debug.assert(pos.wx & mask == 0 and pos.wy & mask == 0 and pos.wz & mask == 0); return chunkCache.find(pos); } }; @@ -307,7 +328,7 @@ pub const ServerWorld = struct { .milliTime = std.time.milliTimestamp(), .lastUnimportantDataSent = std.time.milliTimestamp(), .seed = @bitCast(@as(i64, @truncate(std.time.nanoTimestamp()))), - .name = name, + .name = main.globalAllocator.dupe(u8, name), }; self.itemDropManager.init(main.globalAllocator, self, self.gravity); errdefer self.itemDropManager.deinit(); @@ -355,6 +376,7 @@ pub const ServerWorld = struct { self.itemDropManager.deinit(); self.blockPalette.deinit(); self.wio.deinit(); + main.globalAllocator.free(self.name); main.globalAllocator.destroy(self); } @@ -465,6 +487,10 @@ pub const ServerWorld = struct { return null; } + pub fn getOrGenerateChunk(_: *ServerWorld, pos: chunk.ChunkPosition) *Chunk { + return ChunkManager.getOrGenerateChunk(pos); + } + pub fn getBiome(_: *const ServerWorld, wx: i32, wy: i32, wz: i32) *const terrain.biomes.Biome { const map = terrain.CaveBiomeMap.InterpolatableCaveBiomeMapView.init(.{.wx = wx, .wy = wy, .wz = wz, .voxelSize = 1}, 1); defer map.deinit();