Increase block limit to 4096. Change Schematic format slightly to store extended blockIDs.

InfdevOldWorld now allocates Blocks as a 16-bit array and reads Add into the top 8 bits if present. MCSchematic also allocates Blocks as uint16 and uses WorldEdit's extension to the schematic format to store the Add arrays.

Conflicts:
	test/extended_id_test.py

block_copy.py

@@ -14,7 +14,7 @@ def convertBlocks(destLevel, sourceLevel, blocks, blockData):
 
 def sourceMaskFunc(blocksToCopy):
     if blocksToCopy is not None:
-        typemask = numpy.zeros(256, dtype='bool')
+        typemask = numpy.zeros(materials.id_limit, dtype='bool')
         typemask[blocksToCopy] = 1
 
         def maskedSourceMask(sourceBlocks):

block_fill.py

@@ -1,4 +1,6 @@
 import logging
+from pymclevel import materials
+
 log = logging.getLogger(__name__)
 
 import numpy
@@ -8,7 +10,7 @@ import blockrotation
 from entity import TileEntity
 
 def blockReplaceTable(blocksToReplace):
-    blocktable = numpy.zeros((256, 16), dtype='bool')
+    blocktable = numpy.zeros((materials.id_limit, 16), dtype='bool')
     for b in blocksToReplace:
         if b.hasVariants:
             blocktable[b.ID, b.blockData] = True

blockrotation.py

@@ -1,3 +1,4 @@
+import materials
 from materials import alphaMaterials
 from numpy import arange, zeros
 
@@ -474,9 +475,9 @@ rotationClasses.append(Vines)
 
 
 def masterRotationTable(attrname):
-    # compute a 256x16 table mapping each possible blocktype/data combination to
+    # compute a materials.id_limitx16 table mapping each possible blocktype/data combination to
     # the resulting data when the block is rotated
-    table = zeros((256, 16), dtype='uint8')
+    table = zeros((materials.id_limit, 16), dtype='uint8')
     table[:] = arange(16, dtype='uint8')
     for cls in rotationClasses:
         if hasattr(cls, attrname):

infiniteworld.py

@@ -113,7 +113,7 @@ class AnvilChunkData(object):
         self.root_tag = root_tag
         self.dirty = False
 
-        self.Blocks = zeros((16, 16, world.Height), 'uint8')  # xxx uint16?
+        self.Blocks = zeros((16, 16, world.Height), 'uint16')
         self.Data = zeros((16, 16, world.Height), 'uint8')
         self.BlockLight = zeros((16, 16, world.Height), 'uint8')
         self.SkyLight = zeros((16, 16, world.Height), 'uint8')
@@ -152,6 +152,7 @@ class AnvilChunkData(object):
 
         for sec in self.root_tag["Level"].pop("Sections", []):
             y = sec["Y"].value * 16
+
             for name in "Blocks", "Data", "SkyLight", "BlockLight":
                 arr = getattr(self, name)
                 secarray = sec[name].value
@@ -163,6 +164,11 @@ class AnvilChunkData(object):
 
                 arr[..., y:y + 16] = secarray.swapaxes(0, 2)
 
+            tag = sec.get("Add")
+            if tag is not None:
+                tag.value.shape = (16, 16, 8)
+                add = unpackNibbleArray(tag.value)
+                self.Blocks[...,y:y + 16] |= (array(add, 'uint16') << 8).swapaxes(0, 2)
 
     def savedTagData(self):
         """ does not recalculate any data or light """
@@ -188,7 +194,11 @@ class AnvilChunkData(object):
             BlockLight = packNibbleArray(BlockLight)
             SkyLight = packNibbleArray(SkyLight)
 
-            section['Blocks'] = nbt.TAG_Byte_Array(array(Blocks))
+            add = Blocks >> 8
+            if add.any():
+                section["Add"] = nbt.TAG_Byte_Array(packNibbleArray(add).astype('uint8'))
+
+            section['Blocks'] = nbt.TAG_Byte_Array(array(Blocks, 'uint8'))
             section['Data'] = nbt.TAG_Byte_Array(array(Data))
             section['BlockLight'] = nbt.TAG_Byte_Array(array(BlockLight))
             section['SkyLight'] = nbt.TAG_Byte_Array(array(SkyLight))

materials.py

@@ -53,7 +53,7 @@ class Block(object):
             r = r[self.blockData]
         return r
 
-
+id_limit = 4096
 class MCMaterials(object):
     defaultColor = (0xc9, 0x77, 0xf0, 0xff)
     defaultBrightness = 0
@@ -67,21 +67,22 @@ class MCMaterials(object):
 
         self.defaultName = defaultName
 
-        self.blockTextures = zeros((256, 16, 6, 2), dtype='uint8')
-        self.blockTextures[:] = self.defaultTexture
-        self.names = [[defaultName] * 16 for i in range(256)]
-        self.aka = [[""] * 16 for i in range(256)]
 
-        self.type = [["NORMAL"] * 16] * 256
+        self.blockTextures = zeros((id_limit, 16, 6, 2), dtype='uint8')
+        self.blockTextures[:] = self.defaultTexture
+        self.names = [[defaultName] * 16 for i in range(id_limit)]
+        self.aka = [[""] * 16 for i in range(id_limit)]
+
+        self.type = [["NORMAL"] * 16] * id_limit
         self.blocksByType = defaultdict(list)
         self.allBlocks = []
         self.blocksByID = {}
 
-        self.lightEmission = zeros(256, dtype='uint8')
+        self.lightEmission = zeros(id_limit, dtype='uint8')
         self.lightEmission[:] = self.defaultBrightness
-        self.lightAbsorption = zeros(256, dtype='uint8')
+        self.lightAbsorption = zeros(id_limit, dtype='uint8')
         self.lightAbsorption[:] = self.defaultOpacity
-        self.flatColors = zeros((256, 16, 4), dtype='uint8')
+        self.flatColors = zeros((id_limit, 16, 4), dtype='uint8')
         self.flatColors[:] = self.defaultColor
 
         self.idStr = {}
@@ -153,7 +154,8 @@ class MCMaterials(object):
             import pkg_resources
 
             f = pkg_resources.resource_stream(__name__, filename)
-        except (ImportError, IOError):
+        except (ImportError, IOError), e:
+            print "Cannot get resource_stream for ", filename, e
             root = os.environ.get("PYMCLEVEL_YAML_ROOT", "pymclevel")  # fall back to cwd as last resort
             path = join(root, filename)
 
@@ -752,12 +754,12 @@ pocketMaterials.NetherReactorUsed = pocketMaterials[247, 1]
 #                      b.ID, b.blockData)
 #                  for b in sorted(mats.pocketMaterials.allBlocks)])
 
-_indices = rollaxis(indices((256, 16)), 0, 3)
+_indices = rollaxis(indices((id_limit, 16)), 0, 3)
 
 
 def _filterTable(filters, unavailable, default=(0, 0)):
-    # a filter table is a 256x16 table of (ID, data) pairs.
+    # a filter table is a id_limit table of (ID, data) pairs.
-    table = zeros((256, 16, 2), dtype='uint8')
+    table = zeros((id_limit, 16, 2), dtype='uint8')
     table[:] = _indices
     for u in unavailable:
         try:

mce.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 import mclevelbase
 import mclevel
+import materials
 import infiniteworld
 import sys
 import os
@@ -269,7 +270,7 @@ class mce(object):
         return blockInfo
 
     def readBlocksToCopy(self, command):
-        blocksToCopy = range(256)
+        blocksToCopy = range(materials.id_limit)
         while len(command):
             word = command.pop()
             if word == "noair":
@@ -460,7 +461,7 @@ class mce(object):
             if i % 100 == 0:
                 logging.info("Chunk {0}...".format(i))
 
-        for blockID in range(256):
+        for blockID in range(materials.id_limit):
             block = self.level.materials.blockWithID(blockID, 0)
             if block.hasVariants:
                 for data in range(16):

schematic.py

@@ -17,7 +17,7 @@ from level import MCLevel, EntityLevel
 from materials import alphaMaterials, MCMaterials, namedMaterials
 from mclevelbase import exhaust
 import nbt
-from numpy import array, swapaxes, uint8, zeros
+from numpy import array, swapaxes, uint8, zeros, resize
 
 log = getLogger(__name__)
 
@@ -46,9 +46,6 @@ class MCSchematic (EntityLevel):
         I'm not sure what happens when I try to re-save a rotated schematic.
         """
 
-        # if(shape != None):
-        #    self.setShape(shape)
-
         if filename:
             self.filename = filename
             if None is root_tag and os.path.exists(filename):
@@ -68,7 +65,39 @@ class MCSchematic (EntityLevel):
                 self.materials = namedMaterials[self.Materials]
             else:
                 root_tag["Materials"] = nbt.TAG_String(self.materials.name)
-            self.shapeChunkData()
+
+            w = self.root_tag["Width"].value
+            l = self.root_tag["Length"].value
+            h = self.root_tag["Height"].value
+
+            self._Blocks = self.root_tag["Blocks"].value.astype('uint16').reshape(h, l, w)
+            del self.root_tag["Blocks"]
+            if "AddBlocks" in self.root_tag:
+                # Use WorldEdit's "AddBlocks" array to load and store the 4 high bits of a block ID.
+                # Unlike Minecraft's NibbleArrays, this array stores the first block's bits in the
+                # 4 high bits of the first byte.
+
+                size = (h * l * w)
+
+                # If odd, add one to the size to make sure the adjacent slices line up.
+                add = zeros(size + (size & 1), 'uint16')
+
+                # Fill the even bytes with data
+                add[::2] = self.root_tag["AddBlocks"].value
+
+                # Copy the low 4 bits to the odd bytes
+                add[1::2] = add[::2] & 0xf
+
+                # Shift the even bytes down
+                add[::2] >>= 4
+
+                # Shift every byte up before merging it with Blocks
+                add <<= 8
+                self._Blocks |= add[:size].reshape(h, l, w)
+                del self.root_tag["AddBlocks"]
+
+            self.root_tag["Data"].value = self.root_tag["Data"].value.reshape(h, l, w)
+
 
         else:
             assert shape is not None
@@ -81,12 +110,11 @@ class MCSchematic (EntityLevel):
             root_tag["TileEntities"] = nbt.TAG_List()
             root_tag["Materials"] = nbt.TAG_String(self.materials.name)
 
-            root_tag["Blocks"] = nbt.TAG_Byte_Array(zeros((shape[1], shape[2], shape[0]), uint8))
+            self._Blocks = zeros((shape[1], shape[2], shape[0]), 'uint16')
             root_tag["Data"] = nbt.TAG_Byte_Array(zeros((shape[1], shape[2], shape[0]), uint8))
 
             self.root_tag = root_tag
 
-        self.packUnpack()
         self.root_tag["Data"].value &= 0xF  # discard high bits
 
 
@@ -99,11 +127,33 @@ class MCSchematic (EntityLevel):
 
         self.Materials = self.materials.name
 
-        self.packUnpack()
+        self.root_tag["Blocks"] = nbt.TAG_Byte_Array(self._Blocks.astype('uint8'))
+
+        add = self._Blocks >> 8
+        if add.any():
+            # WorldEdit AddBlocks compatibility.
+            # The first 4-bit value is stored in the high bits of the first byte.
+
+            # Increase odd size by one to align slices.
+            packed_add = zeros(add.size + (add.size & 1), 'uint8')
+            packed_add[:-(add.size & 1)] = add.ravel()
+
+            # Shift even bytes to the left
+            packed_add[::2] <<= 4
+
+            # Merge odd bytes into even bytes
+            packed_add[::2] |= packed_add[1::2]
+
+            # Save only the even bytes, now that they contain the odd bytes in their lower bits.
+            packed_add = packed_add[0::2]
+            self.root_tag["AddBlocks"] = nbt.TAG_Byte_Array(packed_add)
+
         with open(filename, 'wb') as chunkfh:
             self.root_tag.save(chunkfh)
 
-        self.packUnpack()
+        del self.root_tag["Blocks"]
+        self.root_tag.pop("AddBlocks", None)
+
 
     def __str__(self):
         return u"MCSchematic(shape={0}, materials={2}, filename=\"{1}\")".format(self.size, self.filename or u"", self.Materials)
@@ -124,11 +174,11 @@ class MCSchematic (EntityLevel):
 
     @property
     def Blocks(self):
-        return self.root_tag["Blocks"].value
+        return swapaxes(self._Blocks, 0, 2)
 
     @property
     def Data(self):
-        return self.root_tag["Data"].value
+        return swapaxes(self.root_tag["Data"].value, 0, 2)
 
     @property
     def Entities(self):
@@ -152,19 +202,6 @@ class MCSchematic (EntityLevel):
     def _isTagLevel(cls, root_tag):
         return "Schematic" == root_tag.name
 
-    def shapeChunkData(self):
-        w = self.root_tag["Width"].value
-        l = self.root_tag["Length"].value
-        h = self.root_tag["Height"].value
-
-        self.root_tag["Blocks"].value.shape = (h, l, w)
-        self.root_tag["Data"].value.shape = (h, l, w)
-
-    def packUnpack(self):
-        self.root_tag["Blocks"].value = swapaxes(self.root_tag["Blocks"].value, 0, 2)  # yzx to xzy
-        self.root_tag["Data"].value = swapaxes(self.root_tag["Data"].value, 0, 2)  # yzx to xzy
-
-
     def _update_shape(self):
         root_tag = self.root_tag
         shape = self.Blocks.shape
@@ -174,8 +211,8 @@ class MCSchematic (EntityLevel):
 
     def rotateLeft(self):
 
-        self.root_tag["Blocks"].value = swapaxes(self.Blocks, 1, 0)[:, ::-1, :]  # x=z; z=-x
+        self._Blocks = swapaxes(self._Blocks, 1, 0)[:, ::-1, :]  # x=z; z=-x
-        self.root_tag["Data"].value   = swapaxes(self.Data, 1, 0)[:, ::-1, :]  # x=z; z=-x
+        self.root_tag["Data"].value   = swapaxes(self.root_tag["Data"].value, 1, 0)[:, ::-1, :]  # x=z; z=-x
         self._update_shape()
 
         blockrotation.RotateLeft(self.Blocks, self.Data)
@@ -213,20 +250,20 @@ class MCSchematic (EntityLevel):
 
     def roll(self):
         " xxx rotate stuff "
-        self.root_tag["Blocks"].value = swapaxes(self.Blocks, 2, 0)[:, :, ::-1]  # x=z; z=-x
+        self._Blocks = swapaxes(self.Blocks, 2, 0)[:, :, ::-1]  # x=z; z=-x
-        self.root_tag["Data"].value = swapaxes(self.Data, 2, 0)[:, :, ::-1]
+        self.root_tag["Data"].value = swapaxes(self.root_tag["Data"].value, 2, 0)[:, :, ::-1]
         self._update_shape()
 
     def flipVertical(self):
         " xxx delete stuff "
         blockrotation.FlipVertical(self.Blocks, self.Data)
-        self.root_tag["Blocks"].value = self.Blocks[:, :, ::-1]  # y=-y
+        self._Blocks = self.Blocks[:, :, ::-1]  # y=-y
-        self.root_tag["Data"].value = self.Data[:, :, ::-1]
+        self.root_tag["Data"].value = self.root_tag["Data"].value[:, :, ::-1]
 
     def flipNorthSouth(self):
         blockrotation.FlipNorthSouth(self.Blocks, self.Data)
-        self.root_tag["Blocks"].value = self.Blocks[::-1, :, :]  # x=-x
+        self._Blocks = self.Blocks[::-1, :, :]  # x=-x
-        self.root_tag["Data"].value = self.Data[::-1, :, :]
+        self.root_tag["Data"].value = self.root_tag["Data"].value[::-1, :, :]
 
         northSouthPaintingMap = [0, 3, 2, 1]
 
@@ -249,10 +286,9 @@ class MCSchematic (EntityLevel):
             tileEntity["x"].value = self.Width - tileEntity["x"].value - 1
 
     def flipEastWest(self):
-        " xxx flip entities "
         blockrotation.FlipEastWest(self.Blocks, self.Data)
-        self.root_tag["Blocks"].value = self.Blocks[:, ::-1, :]  # z=-z
+        self._Blocks = self._Blocks[:, ::-1, :]  # z=-z
-        self.root_tag["Data"].value = self.Data[:, ::-1, :]
+        self.root_tag["Data"].value = self.root_tag["Data"].value[:, ::-1, :]
 
         eastWestPaintingMap = [2, 1, 0, 3]
 
@@ -271,17 +307,6 @@ class MCSchematic (EntityLevel):
         for tileEntity in self.TileEntities:
             tileEntity["z"].value = self.Length - tileEntity["z"].value - 1
 
-    def setShape(self, shape):
-        """shape is a tuple of (width, height, length).  sets the
-        schematic's properties and clears the block and data arrays"""
-
-        x, y, z = shape
-        shape = (x, z, y)
-
-        self.root_tag["Blocks"].value = zeros(dtype='uint8', shape=shape)
-        self.root_tag["Data"].value = zeros(dtype='uint8', shape=shape)
-        self.shapeChunkData()
-
 
     def setBlockDataAt(self, x, y, z, newdata):
         if x < 0 or y < 0 or z < 0:

test/extended_id_test.py

@@ -1,8 +1,27 @@
 from pymclevel.schematic import MCSchematic
+from pymclevel import MCInfdevOldLevel
+from templevel import TempLevel
 
 __author__ = 'Rio'
 
 def test_schematic_extended_ids():
-    s = MCSchematic(shape=(3, 2, 2))
+    s = MCSchematic(shape=(1, 1, 5))
     s.Blocks[0,0,0] = 2048
+    temp = TempLevel("schematic", createFunc=s.saveToFile)
+    s = temp.level
     assert s.Blocks[0,0,0] == 2048
+
+
+def test_alpha_extended_ids():
+    temp = TempLevel("alpha", createFunc=lambda f: MCInfdevOldLevel(f, create=True))
+    level = temp.level
+    level.createChunk(0, 0)
+
+    level.setBlockAt(0,2,5, 2048)
+    level.saveInPlace()
+    level.close()
+
+    level = MCInfdevOldLevel(filename=level.filename)
+
+    assert level.blockAt(0,2,5) == 2048
+