Add experimental new optimised binary format for undo (saves 33% of file size)

This commit is contained in:
UnknownShadow200 2016-03-25 15:19:23 +11:00
parent 3dff13aade
commit 04198f7f94
9 changed files with 354 additions and 67 deletions

View File

@ -29,7 +29,7 @@ namespace MCGalaxy.Commands
//bla
public override void Use(Player p, string message)
{
if (p != null && (p.isGCMod || p.isMod || p.isDev) && !p.verifiedName) { Player.SendMessage(p, "You can't use GC, because the server hasn't verify-names on"); return; }
if (p != null && !p.verifiedName) { Player.SendMessage(p, "You can't use GC, because the server hasn't verify-names on"); return; }
if (String.IsNullOrEmpty(message)) {
p.InGlobalChat = !p.InGlobalChat;

View File

@ -78,11 +78,9 @@ namespace MCGalaxy.Drawing.Ops {
if (time > End) continue;
if (time < Start) { buffer.CheckIfSend(true); return; }
byte newTile = 0, newExtTile = 0;
item.GetNewExtBlock(out newTile, out newExtTile);
byte tile = 0, extTile = 0;
item.GetExtBlock(out tile, out extTile);
UndoFile.UndoBlock(p, lvl, Pos, timeDelta, buffer, tile, extTile, newTile, newExtTile);
item.GetNewExtBlock(out Pos.newtype, out Pos.newExtType);
item.GetExtBlock(out Pos.type, out Pos.extType);
UndoFile.UndoBlock(p, lvl, Pos, timeDelta, buffer);
}
buffer.CheckIfSend(true);
node = node.Prev;

View File

@ -481,6 +481,7 @@
<Compile Include="Player\Player.Handlers.cs" />
<Compile Include="Player\PlayerInfo.cs" />
<Compile Include="Player\Undo\UndoCache.cs" />
<Compile Include="Player\Undo\UndoFileCBin.cs" />
<Compile Include="Player\Waypoint.cs" />
<Compile Include="Player\Player.Timers.cs" />
<Compile Include="Player\Undo\UndoFile.cs" />

View File

@ -474,7 +474,7 @@ namespace MCGalaxy {
SetPrefix();
playerDb.Dispose();
if (Server.server_owner != "" && Server.server_owner.ToLower().Equals(name.ToLower())) {
if (Server.server_owner != "" && Server.server_owner.CaselessEq(name)) {
if (color == Group.standard.color) color = "&c";
if (title == "") title = "Owner";
SetPrefix();

View File

@ -112,21 +112,17 @@ namespace MCGalaxy.Util {
public void GetExtBlock(out byte type, out byte extType) {
if ((Flags & (1 << 14)) != 0) {
type = Block.custom_block;
extType = Type;
type = Block.custom_block; extType = Type;
} else {
type = Type;
extType = 0;
type = Type; extType = 0;
}
}
public void GetNewExtBlock(out byte type, out byte extType) {
if ((Flags & (1 << 15)) != 0) {
type = Block.custom_block;
extType = NewType;
type = Block.custom_block; extType = NewType;
} else {
type = NewType;
extType = 0;
type = NewType; extType = 0;
}
}
@ -149,6 +145,10 @@ namespace MCGalaxy.Util {
}
return item;
}
public static UndoCacheItem Make(UndoCacheNode node, short timeDelta, Player.UndoPos pos) {
return Make(node, timeDelta, ref pos);
}
}
public sealed class UndoDrawOpEntry {

View File

@ -25,8 +25,9 @@ namespace MCGalaxy.Util {
public abstract class UndoFile {
protected const string undoDir = "extra/undo", prevUndoDir = "extra/undoPrevious";
public static UndoFile OldFormat = new UndoFileText();
public static UndoFile NewFormat = new UndoFileBin();
public static UndoFile TxtFormat = new UndoFileText();
public static UndoFile BinFormat = new UndoFileBin();
public static UndoFile NewFormat = new UndoFileCBin();
protected abstract void SaveUndoData(List<Player.UndoPos> buffer, string path);
@ -57,8 +58,8 @@ namespace MCGalaxy.Util {
Directory.CreateDirectory(playerDir);
int numFiles = Directory.GetFiles(playerDir).Length;
string path = Path.Combine(playerDir, numFiles + NewFormat.Extension);
NewFormat.SaveUndoData(p.UndoBuffer, path);
string path = Path.Combine(playerDir, numFiles + BinFormat.Extension);
BinFormat.SaveUndoData(p.UndoBuffer, path);
}
public static void UndoPlayer(Player p, string target, Vec3U16[] marks, DateTime start, ref bool FoundUser) {
@ -87,7 +88,8 @@ namespace MCGalaxy.Util {
continue;
UndoFile format = null;
if (path.EndsWith(OldFormat.Extension)) format = OldFormat;
if (path.EndsWith(TxtFormat.Extension)) format = TxtFormat;
if (path.EndsWith(BinFormat.Extension)) format = BinFormat;
if (path.EndsWith(NewFormat.Extension)) format = NewFormat;
if (format == null) continue;
@ -115,14 +117,15 @@ namespace MCGalaxy.Util {
}
protected internal static void UndoBlock(Player p, Level lvl, Player.UndoPos Pos,
int timeDelta, BufferedBlockSender buffer,
byte oldType, byte oldExtType, byte newType, byte newExtType) {
Pos.type = lvl.GetTile(Pos.x, Pos.y, Pos.z);
if (Pos.type == newType || Block.Convert(Pos.type) == Block.water
|| Block.Convert(Pos.type) == Block.lava || Pos.type == Block.grass) {
int timeDelta, BufferedBlockSender buffer) {
byte lvlTile = lvl.GetTile(Pos.x, Pos.y, Pos.z);
if (lvlTile == Pos.newtype || Block.Convert(lvlTile) == Block.water
|| Block.Convert(lvlTile) == Block.lava || lvlTile == Block.grass) {
Pos.newtype = oldType; Pos.newExtType = oldExtType;
Pos.extType = newExtType; Pos.timeDelta = timeDelta;
byte newExtType = Pos.newExtType;
Pos.newtype = Pos.type; Pos.newExtType = Pos.extType;
Pos.extType = newExtType; Pos.type = lvlTile;
Pos.timeDelta = timeDelta;
if (lvl.DoBlockchange(p, Pos.x, Pos.y, Pos.z, Pos.newtype, Pos.newExtType)) {
buffer.Add(lvl.PosToInt(Pos.x, Pos.y, Pos.z), Pos.newtype, Pos.newExtType);
buffer.CheckIfSend(false);
@ -131,6 +134,15 @@ namespace MCGalaxy.Util {
}
}
protected static void HighlightBlock(Player p, Level lvl, byte type, ushort x, ushort y, ushort z) {
byte lvlTile = lvl.GetTile(x, y, z);
if (lvlTile == type || Block.Convert(lvlTile) == Block.water || Block.Convert(lvlTile) == Block.lava) {
byte block = (lvlTile == Block.air || Block.Convert(lvlTile) == Block.water
|| Block.Convert(lvlTile) == Block.lava) ? Block.red : Block.green;
p.SendBlockchange(x, y, z, block);
}
}
public static void CreateDefaultDirectories() {
if (!Directory.Exists(undoDir))
Directory.CreateDirectory(undoDir);
@ -152,10 +164,13 @@ namespace MCGalaxy.Util {
for (int i = 0; i < files.Length; i++) {
path = files[i];
if (!path.EndsWith(OldFormat.Extension))
continue;
buffer.Clear();
OldFormat.ReadUndoData(buffer, path);
if (path.EndsWith(BinFormat.Extension)) {
buffer.Clear();
BinFormat.ReadUndoData(buffer, path);
} else if (path.EndsWith(TxtFormat.Extension)) {
buffer.Clear();
TxtFormat.ReadUndoData(buffer, path);
}
string newPath = Path.ChangeExtension(path, NewFormat.Extension);
NewFormat.SaveUndoData(buffer, newPath);

View File

@ -93,7 +93,6 @@ namespace MCGalaxy.Util {
}
protected override void ReadUndoData(List<Player.UndoPos> buffer, string path) {
DateTime now = DateTime.Now;
Player.UndoPos Pos;
using (Stream fs = File.OpenRead(path))
using (BinaryReader r = new BinaryReader(fs))
@ -155,9 +154,9 @@ namespace MCGalaxy.Util {
if (Pos.x < min.X || Pos.y < min.Y || Pos.z < min.Z ||
Pos.x > max.X || Pos.y > max.Y || Pos.z > max.Z) continue;
byte oldType = temp[offset + 8], oldExtType = temp[offset + 9];
byte newType = temp[offset + 10], newExtType = temp[offset + 11];
UndoBlock(p, lvl, Pos, timeDelta, buffer, oldType, oldExtType, newType, newExtType);
Pos.type = temp[offset + 8]; Pos.extType = temp[offset + 9];
Pos.newtype = temp[offset + 10]; Pos.newExtType = temp[offset + 11];
UndoBlock(p, lvl, Pos, timeDelta, buffer);
}
buffer.CheckIfSend(true);
}
@ -189,17 +188,7 @@ namespace MCGalaxy.Util {
DateTime time = chunk.BaseTime.AddTicks(U16(temp, offset + 0) * TimeSpan.TicksPerSecond);
if (time < start) return false;
ushort x = U16(temp, offset + 2), y = U16(temp, offset + 4), z = U16(temp, offset + 6);
byte lvlTile = lvl.GetTile(x, y, z);
byte oldType = temp[offset + 8], oldExtType = temp[offset + 9];
byte newType = temp[offset + 10], newExtType = temp[offset + 11];
if (lvlTile == newType || Block.Convert(lvlTile) == Block.water || Block.Convert(lvlTile) == Block.lava) {
byte block = (lvlTile == Block.air || Block.Convert(lvlTile) == Block.water
|| Block.Convert(lvlTile) == Block.lava) ? Block.red : Block.green;
p.SendBlockchange(x, y, z, block);
}
HighlightBlock(p, lvl, temp[offset + 10], x, y, z);
}
}
}

296
Player/Undo/UndoFileCBin.cs Normal file
View File

@ -0,0 +1,296 @@
/*
Copyright 2015 MCGalaxy
Dual-licensed under the Educational Community License, Version 2.0 and
the GNU General Public License, Version 3 (the "Licenses"); you may
not use this file except in compliance with the Licenses. You may
obtain a copy of the Licenses at
http://www.opensource.org/licenses/ecl2.php
http://www.gnu.org/licenses/gpl-3.0.html
Unless required by applicable law or agreed to in writing,
software distributed under the Licenses are distributed on an "AS IS"
BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
or implied. See the Licenses for the specific language governing
permissions and limitations under the Licenses.
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using MCGalaxy.Drawing;
using MCGalaxy.Levels.IO;
namespace MCGalaxy.Util {
public sealed class UndoFileCBin : UndoFile {
protected override string Extension { get { return ".uncbin"; } }
const int entrySize = 8;
protected override void SaveUndoData(List<Player.UndoPos> buffer, string path) {
UndoCacheNode node = new UndoCacheNode();
string lastLoggedName = "";
using (FileStream fs = File.Create(path)) {
BinaryWriter w = new BinaryWriter(fs);
long entriesPos = 0;
ChunkHeader last = default(ChunkHeader);
foreach (Player.UndoPos uP in buffer) {
DateTime time = Server.StartTime.AddSeconds(uP.timeDelta);
int timeDiff = (int)(time - last.BaseTime).TotalSeconds;
if (last.LevelName != uP.mapName || timeDiff > (65535 >> 2) || last.Entries == ushort.MaxValue) {
if (!LevelInfo.ExistsOffline(uP.mapName)) {
if (uP.mapName != lastLoggedName) {
lastLoggedName = uP.mapName;
Server.s.Log("Missing map file\"" + lastLoggedName+ "\", skipping undo entries");
}
continue;
}
ushort width, height, length;
LvlFile.LoadDimensions(LevelInfo.LevelPath(uP.mapName), out width, out height, out length);
node.Width = width; node.Height = height; node.Length = length;
WriteChunkEntries(w, last.Entries, entriesPos);
node.MapName = uP.mapName;
last = WriteEmptyChunk(w, node, time, ref entriesPos);
}
UndoCacheItem item = UndoCacheItem.Make(node, 0, uP);
int flags = (item.Flags & 0xC000) | timeDiff;
w.Write((ushort)flags); w.Write(item.Index);
w.Write(item.Type); w.Write(item.NewType);
last.Entries++;
}
if (last.Entries > 0)
WriteChunkEntries(w, last.Entries, entriesPos);
}
}
protected override void SaveUndoData(UndoCache buffer, string path) {
using (FileStream fs = File.Create(path)) {
BinaryWriter w = new BinaryWriter(fs);
long entriesPos = 0;
ChunkHeader last = default(ChunkHeader);
UndoCacheNode node = buffer.Tail;
while (node != null) {
List<UndoCacheItem> items = node.Items;
for (int i = 0; i < items.Count; i++) {
UndoCacheItem uP = items[i];
DateTime time = node.BaseTime.AddSeconds(uP.TimeDelta);
int timeDiff = (int)(time - last.BaseTime).TotalSeconds;
if (last.LevelName != node.MapName || timeDiff > (65535 >> 2) || last.Entries == ushort.MaxValue) {
WriteChunkEntries(w, last.Entries, entriesPos);
last = WriteEmptyChunk(w, node, time, ref entriesPos);
}
int flags = (uP.Flags & 0xC000) | timeDiff;
w.Write((ushort)flags); w.Write(uP.Index);
w.Write(uP.Type); w.Write(uP.NewType);
last.Entries++;
}
if (last.Entries > 0)
WriteChunkEntries(w, last.Entries, entriesPos);
node = node.Prev;
}
}
}
protected override void ReadUndoData(List<Player.UndoPos> buffer, string path) {
Player.UndoPos Pos;
UndoCacheItem item = default(UndoCacheItem);
using (Stream fs = File.OpenRead(path))
using (BinaryReader r = new BinaryReader(fs))
{
int approxEntries = (int)(fs.Length / entrySize);
if (buffer.Capacity < approxEntries)
buffer.Capacity = approxEntries;
while (fs.Position < fs.Length) {
ChunkHeader chunk = ReadHeader(fs, r);
Pos.mapName = chunk.LevelName;
for (int j = 0; j < chunk.Entries; j++ ) {
item.Flags = r.ReadUInt16();
DateTime time = chunk.BaseTime.AddTicks((item.Flags & 0x3FFF) * TimeSpan.TicksPerSecond);
Pos.timeDelta = (int)time.Subtract(Server.StartTime).TotalSeconds;
int index = r.ReadInt32();
Pos.x = (ushort)(index % chunk.Width);
Pos.y = (ushort)((index / chunk.Width) / chunk.Length);
Pos.z = (ushort)((index / chunk.Width) % chunk.Length);
item.Type = r.ReadByte();
item.NewType = r.ReadByte();
item.GetExtBlock(out Pos.type, out Pos.extType);
item.GetNewExtBlock(out Pos.newtype, out Pos.newExtType);
buffer.Add(Pos);
}
}
}
}
protected override bool UndoEntry(Player p, string path, Vec3U16[] marks,
ref byte[] temp, DateTime start) {
List<ChunkHeader> list = new List<ChunkHeader>();
int timeDelta = (int)DateTime.UtcNow.Subtract(Server.StartTime).TotalSeconds;
Player.UndoPos Pos = default(Player.UndoPos);
Vec3U16 min = marks[0], max = marks[1];
bool undoArea = min.X != ushort.MaxValue;
UndoCacheItem item = default(UndoCacheItem);
using (Stream fs = File.OpenRead(path))
using (BinaryReader r = new BinaryReader(fs))
{
ReadHeaders(list, r);
for (int i = list.Count - 1; i >= 0; i--) {
ChunkHeader chunk = list[i];
Level lvl;
if (!CheckChunk(chunk, start, p, out lvl))
return false;
if (lvl == null || (p.level != null && !p.level.name.CaselessEq(lvl.name)))
continue;
BufferedBlockSender buffer = new BufferedBlockSender(lvl);
if (!undoArea) {
min = new Vec3U16(0, 0, 0);
max = new Vec3U16((ushort)(lvl.Width - 1), (ushort)(lvl.Height - 1), (ushort)(lvl.Length - 1));
}
Pos.mapName = chunk.LevelName;
fs.Seek(chunk.DataPosition, SeekOrigin.Begin);
if (temp == null) temp = new byte[ushort.MaxValue * entrySize];
fs.Read(temp, 0, chunk.Entries * entrySize);
for (int j = chunk.Entries - 1; j >= 0; j-- ) {
int offset = j * entrySize;
item.Flags = U16(temp, offset + 0);
DateTime time = chunk.BaseTime.AddTicks((item.Flags & 0x3FFF) * TimeSpan.TicksPerSecond);
if (time < start) { buffer.CheckIfSend(true); return false; }
int index = NetUtils.ReadI32(temp, offset + 2);
Pos.x = (ushort)(index % chunk.Width);
Pos.y = (ushort)((index / chunk.Width) / chunk.Length);
Pos.z = (ushort)((index / chunk.Width) % chunk.Length);
if (Pos.x < min.X || Pos.y < min.Y || Pos.z < min.Z ||
Pos.x > max.X || Pos.y > max.Y || Pos.z > max.Z) continue;
item.Type = temp[offset + 6];
item.NewType = temp[offset + 7];
item.GetExtBlock(out Pos.type, out Pos.extType);
item.GetNewExtBlock(out Pos.newtype, out Pos.newExtType);
UndoBlock(p, lvl, Pos, timeDelta, buffer);
}
buffer.CheckIfSend(true);
}
}
return true;
}
protected override bool HighlightEntry(Player p, string path,
ref byte[] temp, DateTime start) {
List<ChunkHeader> list = new List<ChunkHeader>();
using (Stream fs = File.OpenRead(path))
using (BinaryReader r = new BinaryReader(fs))
{
ReadHeaders(list, r);
for (int i = list.Count - 1; i >= 0; i--) {
ChunkHeader chunk = list[i];
Level lvl;
if (!CheckChunk(chunk, start, p, out lvl))
return false;
if (lvl == null || lvl != p.level) continue;
fs.Seek(chunk.DataPosition, SeekOrigin.Begin);
if (temp == null) temp = new byte[ushort.MaxValue * entrySize];
fs.Read(temp, 0, chunk.Entries * entrySize);
for (int j = chunk.Entries - 1; j >= 0; j-- ) {
int offset = j * entrySize;
ushort flags = U16(temp, offset + 0);
DateTime time = chunk.BaseTime.AddTicks((flags & 0x3FFF) * TimeSpan.TicksPerSecond);
if (time < start) return false;
int index = NetUtils.ReadI32(temp, offset + 2);
ushort x = (ushort)(index % chunk.Width);
ushort y = (ushort)((index / chunk.Width) / chunk.Length);
ushort z = (ushort)((index / chunk.Width) % chunk.Length);
HighlightBlock(p, lvl, temp[offset + 7], x, y, z);
}
}
}
return true;
}
static ushort U16(byte[] buffer, int offset) {
return (ushort)(buffer[offset + 0] | buffer[offset + 1] << 8);
}
static bool CheckChunk(ChunkHeader chunk, DateTime start, Player p, out Level lvl) {
DateTime time = chunk.BaseTime;
lvl = null;
if (time.AddTicks((65536 >> 2) * TimeSpan.TicksPerSecond) < start)
return false; // we can safely discard the entire chunk
lvl = LevelInfo.FindExact(chunk.LevelName);
return true;
}
struct ChunkHeader {
public string LevelName;
public DateTime BaseTime;
public ushort Entries;
public ushort Width, Height, Length;
public long DataPosition;
}
static void ReadHeaders(List<ChunkHeader> list, BinaryReader r) {
Stream s = r.BaseStream;
long len = s.Length;
while (s.Position < len) {
ChunkHeader header = ReadHeader(s, r);
s.Seek(header.Entries * entrySize, SeekOrigin.Current);
list.Add(header);
}
}
static ChunkHeader ReadHeader(Stream s, BinaryReader r) {
ChunkHeader header = default(ChunkHeader);
byte[] mapNameData = r.ReadBytes(r.ReadUInt16());
header.LevelName = Encoding.UTF8.GetString(mapNameData);
header.BaseTime = new DateTime(r.ReadInt64(), DateTimeKind.Utc);
header.Entries = r.ReadUInt16();
header.Width = r.ReadUInt16();
header.Height = r.ReadUInt16();
header.Length = r.ReadUInt16();
header.DataPosition = s.Position;
return header;
}
static void WriteChunkEntries(BinaryWriter w, ushort entries, long entriesPos) {
long curPos = w.BaseStream.Position;
w.BaseStream.Seek(entriesPos, SeekOrigin.Begin);
w.Write(entries);
w.BaseStream.Seek(curPos, SeekOrigin.Begin);
}
static ChunkHeader WriteEmptyChunk(BinaryWriter w, UndoCacheNode node, DateTime time, ref long entriesPos) {
ChunkHeader header = default(ChunkHeader);
time = time.ToUniversalTime();
byte[] mapBytes = Encoding.UTF8.GetBytes(node.MapName);
w.Write((ushort)mapBytes.Length);
w.Write(mapBytes); header.LevelName = node.MapName;
w.Write(time.Ticks); header.BaseTime = time;
entriesPos = w.BaseStream.Position;
w.Write((ushort)0);
w.Write((ushort)node.Width); header.Width = (ushort)node.Width;
w.Write((ushort)node.Height); header.Height = (ushort)node.Height;
w.Write((ushort)node.Length); header.Length = (ushort)node.Length;
return header;
}
}
}

View File

@ -103,9 +103,9 @@ namespace MCGalaxy.Util {
if (Pos.x < min.X || Pos.y < min.Y || Pos.z < min.Z ||
Pos.x > max.X || Pos.y > max.Y || Pos.z > max.Z) continue;
byte newType = Convert.ToByte(lines[(i * 7) - 1]);
byte oldType = Convert.ToByte(lines[(i * 7) - 2]);
UndoBlock(p, lvl, Pos, timeDelta, buffer, oldType, 0, newType, 0);
Pos.newtype = Convert.ToByte(lines[(i * 7) - 1]);
Pos.type = Convert.ToByte(lines[(i * 7) - 2]);
UndoBlock(p, lvl, Pos, timeDelta, buffer);
} catch {
}
}
@ -114,31 +114,19 @@ namespace MCGalaxy.Util {
}
protected override bool HighlightEntry(Player p, string path, ref byte[] temp, DateTime start) {
Player.UndoPos Pos;
Pos.extType = 0; Pos.newExtType = 0;
string[] lines = File.ReadAllText(path).Split(' ');
// because we have space to end of each entry, need to subtract one otherwise we'll start at a "".
for (int i = (lines.Length - 1) / 7; i >= 0; i--) {
try {
// line format: mapName x y z date oldblock newblock
if (!InTime(lines[(i * 7) - 3], start)) return false;
Level foundLevel = LevelInfo.FindExact(lines[(i * 7) - 7]);
if (foundLevel == null || foundLevel != p.level) continue;
Level lvl = LevelInfo.FindExact(lines[(i * 7) - 7]);
if (lvl == null || lvl != p.level) continue;
Pos.mapName = foundLevel.name;
Pos.x = Convert.ToUInt16(lines[(i * 7) - 6]);
Pos.y = Convert.ToUInt16(lines[(i * 7) - 5]);
Pos.z = Convert.ToUInt16(lines[(i * 7) - 4]);
Pos.type = foundLevel.GetTile(Pos.x, Pos.y, Pos.z);
if (Pos.type == Convert.ToByte(lines[(i * 7) - 1]) ||
Block.Convert(Pos.type) == Block.water || Block.Convert(Pos.type) == Block.lava) {
if (Pos.type == Block.air || Block.Convert(Pos.type) == Block.water || Block.Convert(Pos.type) == Block.lava)
p.SendBlockchange(Pos.x, Pos.y, Pos.z, Block.red);
else
p.SendBlockchange(Pos.x, Pos.y, Pos.z, Block.green);
}
ushort x = Convert.ToUInt16(lines[(i * 7) - 6]);
ushort y = Convert.ToUInt16(lines[(i * 7) - 5]);
ushort z = Convert.ToUInt16(lines[(i * 7) - 4]);
HighlightBlock(p, lvl, Convert.ToByte(lines[(i * 7) - 1]), x, y, z);
} catch { }
}
return true;