Merge pull request #110 from dcoshea/sign_coding_and_mem_leak

Fixes for Anvil memory leak and UTF-8 coding in sign text

infiniteworld.py

@@ -720,6 +720,9 @@ class InfdevChunk(LightedChunk):
         root_tag.save(buf=buf)
         return deflate(buf.getvalue())
 
+    def _discardUncompressed(self):
+        self.root_tag = None
+
     def _compressChunk(self):
         root_tag = self.root_tag
         if root_tag is None:
@@ -730,7 +733,7 @@ class InfdevChunk(LightedChunk):
         if self.compressMode == MCRegionFile.VERSION_DEFLATE:
             self.compressedTag = self.compressTagDeflate(root_tag)
 
-        self.root_tag = None
+        self._discardUncompressed()
 
     def decompressTagGzip(self, data):
         return nbt.load(buf=nbt.gunzip(data))
@@ -774,7 +777,7 @@ class InfdevChunk(LightedChunk):
         if not self.dirty:
             # if we are not dirty, just throw the
             # uncompressed tag structure away. rely on the OS disk cache.
-            self.root_tag = None
+            self._discardUncompressed()
         else:
             if self.root_tag is not None:
                 self.sanitizeBlocks()  # xxx
@@ -799,6 +802,8 @@ class InfdevChunk(LightedChunk):
             try:
                 self._decompressChunk()
 
+            except MemoryError:
+                raise
             except Exception, e:
                 error(u"Malformed NBT data in file: {0} ({1})".format(self.filename, e))
                 if self.world:
@@ -884,6 +889,8 @@ class InfdevChunk(LightedChunk):
                 self.dataIsPacked = True
                 self.decompress()
 
+            except MemoryError:
+                raise
             except Exception, e:
                 error(u"Incorrect chunk format in file: {0} ({1})".format(self.filename, e))
                 if self.world:
@@ -1109,6 +1116,14 @@ class AnvilChunk(InfdevChunk):
 
                 arr[..., y:y + 16] = secarray.swapaxes(0, 2)
 
+    def _discardUncompressed(self):
+        self._Blocks = None
+        self._Data = None
+        self._BlockLight = None
+        self._SkyLight = None
+
+        super(AnvilChunk, self)._discardUncompressed()
+
     def _compressChunk(self):
         sections = self.root_tag[Level][Sections] = nbt.TAG_List()
 
@@ -2731,6 +2746,8 @@ class MCInfdevOldLevel(ChunkedLevelMixin, EntityLevel):
                     data = nbt.gunzip(cdata)
                     chunk.root_tag = nbt.load(buf=data)
 
+        except MemoryError:
+            raise
         except Exception, e:
             raise ChunkMalformed("Chunk {0} had an error: {1!r}".format(chunk.chunkPosition, e), sys.exc_info()[2])
 

mce.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+import mclevelbase
 import mclevel
 import infiniteworld
 import sys
@@ -604,7 +605,15 @@ class mce(object):
         else:
             filename = self.level.displayName + ".signs"
 
-        outFile = codecs.open(filename, "w", encoding='utf-8')
+        # It appears that Minecraft interprets the sign text as UTF-8,
+        # so we should decode it as such too.
+        decodeSignText = codecs.getdecoder('utf-8')
+        # We happen to encode the output file in UTF-8 too, although
+        # we could use another UTF encoding.  The '-sig' encoding puts
+        # a signature at the start of the output file that tools such
+        # as Microsoft Windows Notepad and Emacs understand to mean
+        # the file has UTF-8 encoding.
+        outFile = codecs.open(filename, "w", encoding='utf-8-sig')
 
         print "Dumping signs..."
         signCount = 0
@@ -612,7 +621,7 @@ class mce(object):
         for i, cPos in enumerate(self.level.allChunks):
             try:
                 chunk = self.level.getChunk(*cPos)
-            except mclevel.ChunkMalformed:
+            except mclevelbase.ChunkMalformed:
                 continue
 
             for tileEntity in chunk.TileEntities:
@@ -621,7 +630,8 @@ class mce(object):
 
                     outFile.write(str(map(lambda x: tileEntity[x].value, "xyz")) + "\n")
                     for i in range(4):
-                        outFile.write(tileEntity["Text{0}".format(i + 1)].value + u"\n")
+                        signText = tileEntity["Text{0}".format(i + 1)].value
+                        outFile.write(decodeSignText(signText)[0] + u"\n")
 
             if i % 100 == 0:
                 print "Chunk {0}...".format(i)
@@ -770,7 +780,7 @@ class mce(object):
         for i, cPos in enumerate(self.level.allChunks):
             try:
                 chunk = self.level.getChunk(*cPos)
-            except mclevel.ChunkMalformed:
+            except mclevelbase.ChunkMalformed:
                 continue
 
             for tileEntity in chunk.TileEntities:

schematic.py

@@ -132,6 +132,8 @@ class MCSchematic (EntityLevel):
 
         try:
             self.root_tag = nbt.load(buf=data)
+        except MemoryError:
+             raise
         except Exception, e:
             error(u"Malformed NBT data in schematic file: {0} ({1})".format(self.filename, e))
             raise ChunkMalformed((e, self.filename), sys.exc_info()[2])