Refactor: Move copyBlocksFrom and supporting methods to its own file and simplify them.

Add a function to ChunkBase for getting the slices for a chunk's arrays and the area of the chunk bounded by a given BoundingBox
Remove copyBlocksFromFinite and use copyBlocksFromInfinite as the default implementation
Rewrite copyBlocksFrom to use chunk.getChunkSlicesForBox instead of world.getChunkSlices

block_copy.py

@@ -0,0 +1,150 @@
+from datetime import datetime
+import logging
+import numpy
+from box import BoundingBox
+from mclevelbase import exhaust
+import materials
+
+log = logging.getLogger(__name__)
+
+def convertBlocks(destLevel, sourceLevel, blocks, blockData):
+    return materials.convertBlocks(destLevel.materials, sourceLevel.materials, blocks, blockData)
+
+def sourceMaskFunc(blocksToCopy):
+    if blocksToCopy is not None:
+        typemask = numpy.zeros(256, dtype='bool')
+        typemask[blocksToCopy] = 1
+
+        def maskedSourceMask(sourceBlocks):
+            return typemask[sourceBlocks]
+
+        return maskedSourceMask
+
+    def unmaskedSourceMask(_sourceBlocks):
+        return slice(None, None)
+
+    return unmaskedSourceMask
+
+
+def adjustCopyParameters(destLevel, sourceLevel, sourceBox, destinationPoint):
+    # if the destination box is outside the level, it and the source corners are moved inward to fit.
+    # ValueError is raised if the source corners are outside sourceLevel
+    (x, y, z) = map(int, destinationPoint)
+
+    sourceBox = BoundingBox(sourceBox.origin, sourceBox.size)
+
+    (lx, ly, lz) = sourceBox.size
+    log.debug(u"Asked to copy {0} blocks \n\tfrom {1} in {3}\n\tto {2} in {4}" .format(ly * lz * lx, sourceBox, destinationPoint, sourceLevel, destLevel))
+
+    # clip the source ranges to this level's edges.  move the destination point as needed.
+    # xxx abstract this
+    if y < 0:
+        sourceBox.origin[1] -= y
+        sourceBox.size[1] += y
+        y = 0
+    if y + sourceBox.size[1] > destLevel.Height:
+        sourceBox.size[1] -= y + sourceBox.size[1] - destLevel.Height
+        y = destLevel.Height - sourceBox.size[1]
+
+    # for infinite levels, don't clip along those dimensions because the
+    # infinite copy func will just skip missing chunks
+    if destLevel.Width != 0:
+        if x < 0:
+            sourceBox.origin[0] -= x
+            sourceBox.size[0] += x
+            x = 0
+        if x + sourceBox.size[0] > destLevel.Width:
+            sourceBox.size[0] -= x + sourceBox.size[0] - destLevel.Width
+            # x=self.Width-sourceBox.size[0]
+
+    if destLevel.Length != 0:
+        if z < 0:
+            sourceBox.origin[2] -= z
+            sourceBox.size[2] += z
+            z = 0
+        if z + sourceBox.size[2] > destLevel.Length:
+            sourceBox.size[2] -= z + sourceBox.size[2] - destLevel.Length
+            # z=self.Length-sourceBox.size[2]
+
+    destinationPoint = (x, y, z)
+
+    return sourceBox, destinationPoint
+
+
+def copyBlocksFromIter(destLevel, sourceLevel, sourceBox, destinationPoint, blocksToCopy=None, entities=True, create=False):
+    """ copy blocks between two infinite levels by looping through the
+    destination's chunks. make a sub-box of the source level for each chunk
+    and copy block and entities in the sub box to the dest chunk."""
+
+    (lx, ly, lz) = sourceBox.size
+
+    sourceBox, destinationPoint = adjustCopyParameters(destLevel, sourceLevel, sourceBox, destinationPoint)
+    # needs work xxx
+    log.info(u"Copying {0} blocks from {1} to {2}" .format(ly * lz * lx, sourceBox, destinationPoint))
+    startTime = datetime.now()
+
+    destBox = BoundingBox(destinationPoint, sourceBox.size)
+    chunkCount = destBox.chunkCount
+    i = 0
+    sourceMask = sourceMaskFunc(blocksToCopy)
+
+    copyOffset = [d - s for s, d in zip(sourceBox.origin, destinationPoint)]
+
+    # Visit each chunk in the destination area.
+    #   Get the region of the source area corresponding to that chunk
+    #   Visit each chunk of the region of the source area
+    #     Get the slices of the destination chunk
+    #     Get the slices of the source chunk
+    #     Copy blocks and data
+
+    for destCpos in destBox.chunkPositions:
+        cx, cz = destCpos
+
+        destChunkBox = BoundingBox((cx << 4, 0, cz << 4), (16, destLevel.Height, 16)).intersect(destBox)
+        destChunkBoxInSourceLevel = BoundingBox([d - o for o, d in zip(copyOffset, destChunkBox.origin)], destChunkBox.size)
+
+        if not destLevel.containsChunk(*destCpos):
+            if create and any(sourceLevel.containsChunk(*c) for c in destChunkBoxInSourceLevel.chunkPositions):
+                # Only create chunks in the destination level if the source level has chunks covering them.
+                destLevel.createChunk(*destCpos)
+            else:
+                continue
+
+        destChunk = destLevel.getChunk(*destCpos)
+
+
+        i += 1
+        yield (i, chunkCount)
+        if i % 100 == 0:
+            log.info("Chunk {0}...".format(i))
+
+        for srcCpos in destChunkBoxInSourceLevel.chunkPositions:
+            if not sourceLevel.containsChunk(*srcCpos):
+                continue
+
+            sourceChunk = sourceLevel.getChunk(*srcCpos)
+
+            sourceChunkBox, sourceSlices = sourceChunk.getChunkSlicesForBox(destChunkBoxInSourceLevel)
+            sourceChunkBoxInDestLevel = BoundingBox([d + o for o, d in zip(copyOffset, sourceChunkBox.origin)], sourceChunkBox.size)
+
+            _, destSlices = destChunk.getChunkSlicesForBox(sourceChunkBoxInDestLevel)
+
+            sourceBlocks = sourceChunk.Blocks[sourceSlices]
+            sourceData = sourceChunk.Data[sourceSlices]
+
+            mask = sourceMask(sourceBlocks)
+            convertedSourceBlocks, convertedSourceData = convertBlocks(destLevel, sourceLevel, sourceBlocks, sourceData)
+
+            destChunk.Blocks[destSlices][mask] = convertedSourceBlocks[mask]
+            if convertedSourceData is not None:
+                destChunk.Data[destSlices][mask] = convertedSourceData[mask]
+
+        destChunk.chunkChanged()
+
+    for i in destLevel.copyEntitiesFromIter(sourceLevel, sourceBox, destinationPoint, entities):
+        yield i
+
+    log.info("Duration: {0}".format(datetime.now() - startTime))
+
+def copyBlocksFrom(destLevel, sourceLevel, sourceBox, destinationPoint, blocksToCopy=None, entities=True, create=False):
+    return exhaust(copyBlocksFromIter(destLevel, sourceLevel, sourceBox, destinationPoint, blocksToCopy, entities, create))

infiniteworld.py

@@ -493,166 +493,8 @@ class ChunkedLevelMixin(MCLevel):
             skyLight[xInChunk, zInChunk, y] = lightValue
         return oldValue < lightValue
 
-    def sourceMaskFunc(self, blocksToCopy):
-        if blocksToCopy is not None:
-            typemask = zeros(256, dtype='bool')
-            typemask[blocksToCopy] = 1
-
-            def maskedSourceMask(sourceBlocks):
-                return typemask[sourceBlocks]
-
-            return maskedSourceMask
-
-        def unmaskedSourceMask(_sourceBlocks):
-            return slice(None, None)
-
-        return unmaskedSourceMask
-
     createChunk = NotImplemented
 
-    def copyBlocksFromFiniteIter(self, sourceLevel, sourceBox, destinationPoint, blocksToCopy, create=False):
-        # assumes destination point and bounds have already been checked.
-        (sx, sy, sz) = sourceBox.origin
-
-        start = datetime.now()
-
-        sourceMask = self.sourceMaskFunc(blocksToCopy)
-
-        destBox = BoundingBox(destinationPoint, sourceBox.size)
-
-        i = 0
-        chunkCount = float(destBox.chunkCount)
-
-        for (cPos, slices, point) in self._getSlices(destBox):
-            if not self.containsChunk(*cPos):
-                if create:
-                    self.createChunk(*cPos)
-                else:
-                    continue
-            chunk = self.getChunk(*cPos)
-
-            i += 1
-            yield (i, chunkCount)
-
-            if i % 100 == 0:
-                info("Chunk {0}...".format(i))
-
-            blocks = chunk.Blocks[slices]
-
-            localSourceCorner2 = (
-                sx + point[0] + blocks.shape[0],
-                sy + blocks.shape[2],
-                sz + point[2] + blocks.shape[1],
-            )
-
-            sourceBlocks = sourceLevel.Blocks[sx + point[0]:localSourceCorner2[0],
-                                              sz + point[2]:localSourceCorner2[2],
-                                              sy:localSourceCorner2[1]]
-            # sourceBlocks = filterTable[sourceBlocks]
-            mask = sourceMask(sourceBlocks)
-
-            # for small level slices, reduce the destination area
-            x, z, y = sourceBlocks.shape
-            blocks = blocks[0:x, 0:z, 0:y]
-
-            sourceData = None
-            if hasattr(sourceLevel, 'Data'):
-                # indev or schematic
-                sourceData = sourceLevel.Data[sx + point[0]:localSourceCorner2[0],
-                                              sz + point[2]:localSourceCorner2[2],
-                                              sy:localSourceCorner2[1]]
-
-            data = chunk.Data[slices][0:x, 0:z, 0:y]
-
-            convertedSourceBlocks, convertedSourceData = self.convertBlocksFromLevel(sourceLevel, sourceBlocks, sourceData)
-
-            blocks[mask] = convertedSourceBlocks[mask]
-            if convertedSourceData is not None:
-                data[mask] = (convertedSourceData[:, :, :])[mask]
-                data[mask] &= 0xf
-
-            chunk.chunkChanged()
-
-        d = datetime.now() - start
-        if i:
-            info("Finished {2} chunks in {0} ({1} per chunk)".format(d, d / i, i))
-
-    def copyBlocksFromInfiniteIter(self, sourceLevel, sourceBox, destinationPoint, blocksToCopy, create=False):
-        """ copy blocks between two infinite levels by looping through the
-        destination's chunks. make a sub-box of the source level for each chunk
-        and copy block and entities in the sub box to the dest chunk."""
-
-        # assumes destination point and bounds have already been checked.
-        destBox = BoundingBox(destinationPoint, sourceBox.size)
-        chunkCount = destBox.chunkCount
-        i = 0
-        sourceMask = self.sourceMaskFunc(blocksToCopy)
-
-        def subbox(slices, point):
-            size = [s.stop - s.start for s in slices]
-            size[1], size[2] = size[2], size[1]
-            return BoundingBox([p + a for p, a in zip(point, sourceBox.origin)], size)
-
-        def shouldCreateFunc(slices, point):
-            box = subbox(slices, point)
-            b = any(list(sourceLevel.containsChunk(*c) for c in box.chunkPositions))  # any() won't take a generator-expression :(
-            # if b == False:
-            #    print 'Skipped ', list(box.chunkPositions)
-            return b
-
-        for cPos, slices, point in self._getSlices(destBox):
-            if not self.containsChunk(*cPos):
-                if shouldCreateFunc(slices, point):
-                    self.createChunk(*cPos)
-                else:
-                    continue
-            chunk = self.getChunk(*cPos)
-
-            i += 1
-            yield (i, chunkCount)
-            if i % 100 == 0:
-                info("Chunk {0}...".format(i))
-
-            dstblocks = chunk.Blocks[slices]
-            dstdata = chunk.Data[slices]
-            sourceSubBox = subbox(slices, point)
-            for srcchunk, srcslices, srcpoint in sourceLevel.getChunkSlices(sourceSubBox):
-                srcpoint = srcpoint[0], srcpoint[2], srcpoint[1]
-                sourceBlocks = srcchunk.Blocks[srcslices]
-                sourceData = srcchunk.Data[srcslices]
-                mask = sourceMask(sourceBlocks)
-                convertedSourceBlocks, convertedSourceData = self.convertBlocksFromLevel(sourceLevel, sourceBlocks, sourceData)
-
-                dstslices = [slice(p, p + (s.stop - s.start)) for p, s in zip(srcpoint, srcslices)]
-                dstblocks[dstslices][mask] = convertedSourceBlocks[mask]
-                if convertedSourceData is not None:
-                    dstdata[dstslices][mask] = convertedSourceData[mask]
-
-            chunk.chunkChanged()
-
-    def copyBlocksFrom(self, sourceLevel, sourceBox, destinationPoint, blocksToCopy=None, entities=True, create=False):
-        return exhaust(self.copyBlocksFromIter(sourceLevel, sourceBox, destinationPoint, blocksToCopy, entities, create))
-
-    def copyBlocksFromIter(self, sourceLevel, sourceBox, destinationPoint, blocksToCopy=None, entities=True, create=False):
-        (lx, ly, lz) = sourceBox.size
-
-        sourceBox, destinationPoint = self.adjustCopyParameters(sourceLevel, sourceBox, destinationPoint)
-        # needs work xxx
-        info(u"Copying {0} blocks from {1} to {2}" .format(ly * lz * lx, sourceBox, destinationPoint))
-        startTime = datetime.now()
-
-        if not sourceLevel.isInfinite:
-            for i in self.copyBlocksFromFiniteIter(sourceLevel, sourceBox, destinationPoint, blocksToCopy, create):
-                yield i
-
-        else:
-            for i in self.copyBlocksFromInfiniteIter(sourceLevel, sourceBox, destinationPoint, blocksToCopy, create):
-                yield i
-
-        for i in self.copyEntitiesFromIter(sourceLevel, sourceBox, destinationPoint, entities):
-            yield i
-        info("Duration: {0}".format(datetime.now() - startTime))
-
     def fillBlocks(self, box, blockInfo, blocksToReplace=()):
         return exhaust(self.fillBlocksIter(box, blockInfo, blocksToReplace))
 
@@ -1573,8 +1415,11 @@ class MCInfdevOldLevel(ChunkedLevelMixin, EntityLevel):
         chunkData = self._loadedChunkData.get((cx, cz))
         if chunkData is not None: return chunkData
 
+        data = self._getChunkBytes(cx, cz)
+        if data is None:
+            raise ChunkNotPresent, (cx, cz)
+
         try:
-            data = self._getChunkBytes(cx, cz)
             root_tag = nbt.load(buf=data)
         except MemoryError:
             raise

level.py

@@ -433,134 +433,8 @@ class MCLevel(object):
 
     # --- Copying ---
 
-    def copyBlocksFromFiniteToFinite(self, sourceLevel, sourceBox, destinationPoint, blocksToCopy):
+    from block_copy import copyBlocksFrom, copyBlocksFromIter
-        # assume destinationPoint is entirely within this level, and the size of sourceBox fits entirely within it.
-        sourcex, sourcey, sourcez = map(slice, sourceBox.origin, sourceBox.maximum)
-        destCorner2 = map(lambda a, b: a + b, sourceBox.size, destinationPoint)
-        destx, desty, destz = map(slice, destinationPoint, destCorner2)
 
-        sourceData = None
-        if hasattr(sourceLevel, 'Data'):
-            sourceData = sourceLevel.Data[sourcex, sourcez, sourcey]
-        convertedSourceBlocks, convertedSourceData = self.convertBlocksFromLevel(sourceLevel, sourceLevel.Blocks[sourcex, sourcez, sourcey], sourceData)
-
-        blocks = self.Blocks[destx, destz, desty]
-
-        mask = slice(None, None)
-
-        if not (blocksToCopy is None):
-            typemask = zeros(256, dtype='bool')
-            typemask[blocksToCopy] = True
-            mask = typemask[convertedSourceBlocks]
-
-        blocks[mask] = convertedSourceBlocks[mask]
-        if hasattr(self, 'Data') and hasattr(sourceLevel, 'Data'):
-            data = self.Data[destx, destz, desty]
-            data[mask] = convertedSourceData[mask]
-
-    def copyBlocksFromInfinite(self, sourceLevel, sourceBox, destinationPoint, blocksToCopy):
-        return exhaust(self.copyBlocksFromInfinite(sourceLevel, sourceBox, destinationPoint, blocksToCopy))
-
-    def copyBlocksFromInfiniteIter(self, sourceLevel, sourceBox, destinationPoint, blocksToCopy):
-        if blocksToCopy is not None:
-            typemask = zeros(256, dtype='bool')
-            typemask[blocksToCopy] = True
-
-        for i, (chunk, slices, point) in enumerate(sourceLevel.getChunkSlices(sourceBox)):
-            point = map(lambda a, b: a + b, point, destinationPoint)
-            point = point[0], point[2], point[1]
-            mask = slice(None, None)
-
-            convertedSourceBlocks, convertedSourceData = self.convertBlocksFromLevel(sourceLevel, chunk.Blocks[slices], chunk.Data[slices])
-
-            destSlices = [slice(p, p + s.stop - s.start) for p, s in zip(point, slices)]
-
-            blocks = self.Blocks[destSlices]
-
-            if blocksToCopy is not None:
-                mask = typemask[convertedSourceBlocks]
-
-            blocks[mask] = convertedSourceBlocks[mask]
-
-            if hasattr(self, 'Data'):
-                data = self.Data[destSlices]
-                data[mask] = convertedSourceData[mask]
-
-            yield i
-
-    def adjustCopyParameters(self, sourceLevel, sourceBox, destinationPoint):
-
-        # if the destination box is outside the level, it and the source corners are moved inward to fit.
-        # ValueError is raised if the source corners are outside sourceLevel
-        (x, y, z) = map(int, destinationPoint)
-
-        sourceBox = BoundingBox(sourceBox.origin, sourceBox.size)
-
-        (lx, ly, lz) = sourceBox.size
-        debug(u"Asked to copy {0} blocks \n\tfrom {1} in {3}\n\tto {2} in {4}" .format(ly * lz * lx, sourceBox, destinationPoint, sourceLevel, self))
-
-        # clip the source ranges to this level's edges.  move the destination point as needed.
-        # xxx abstract this
-        if y < 0:
-            sourceBox.origin[1] -= y
-            sourceBox.size[1] += y
-            y = 0
-        if y + sourceBox.size[1] > self.Height:
-            sourceBox.size[1] -= y + sourceBox.size[1] - self.Height
-            y = self.Height - sourceBox.size[1]
-
-        # for infinite levels, don't clip along those dimensions because the
-        # infinite copy func will just skip missing chunks
-        if self.Width != 0:
-            if x < 0:
-                sourceBox.origin[0] -= x
-                sourceBox.size[0] += x
-                x = 0
-            if x + sourceBox.size[0] > self.Width:
-                sourceBox.size[0] -= x + sourceBox.size[0] - self.Width
-                # x=self.Width-sourceBox.size[0]
-
-        if self.Length != 0:
-            if z < 0:
-                sourceBox.origin[2] -= z
-                sourceBox.size[2] += z
-                z = 0
-            if z + sourceBox.size[2] > self.Length:
-                sourceBox.size[2] -= z + sourceBox.size[2] - self.Length
-                # z=self.Length-sourceBox.size[2]
-
-        destinationPoint = (x, y, z)
-
-        return sourceBox, destinationPoint
-
-    def copyBlocksFrom(self, sourceLevel, sourceBox, destinationPoint, blocksToCopy=None, entities=True, create=False):
-        return exhaust(self.copyBlocksFromIter(sourceLevel, sourceBox, destinationPoint, blocksToCopy, entities, create))
-
-    def copyBlocksFromIter(self, sourceLevel, sourceBox, destinationPoint, blocksToCopy=None, entities=True, create=False):
-        if (not sourceLevel.isInfinite) and not(
-               sourceLevel.containsPoint(*sourceBox.origin) and
-               sourceLevel.containsPoint(*map(lambda x: x - 1, sourceBox.maximum))):
-            raise ValueError("{0} cannot provide blocks between {1}".format(sourceLevel, sourceBox))
-
-        sourceBox, destinationPoint = self.adjustCopyParameters(sourceLevel, sourceBox, destinationPoint)
-        yield
-
-        if min(sourceBox.size) <= 0:
-            print "Empty source box, aborting"
-            return
-
-        info(u"Copying {0} blocks from {1} to {2}" .format(sourceBox.volume, sourceBox, destinationPoint))
-
-        if not sourceLevel.isInfinite:
-            self.copyBlocksFromFiniteToFinite(sourceLevel, sourceBox, destinationPoint, blocksToCopy)
-        else:
-            for i in self.copyBlocksFromInfiniteIter(sourceLevel, sourceBox, destinationPoint, blocksToCopy):
-                yield i
-        for i in self.copyEntitiesFromIter(sourceLevel, sourceBox, destinationPoint, entities):
-            yield i
-
-    def convertBlocksFromLevel(self, sourceLevel, blocks, blockData):
-        return materials.convertBlocks(self.materials, sourceLevel.materials, blocks, blockData)
 
     def saveInPlace(self):
         self.saveToFile(self.filename)
@@ -792,6 +666,23 @@ class ChunkBase(EntityLevel):
         return self.world.materials
 
 
+    def getChunkSlicesForBox(self, box):
+        """
+         Given a BoundingBox enclosing part of the world, return a smaller box enclosing the part of this chunk
+         intersecting the given box, and a tuple of slices that can be used to select the corresponding parts
+         of this chunk's block and data arrays.
+        """
+        bounds = self.bounds
+        localBox = box.intersect(bounds)
+
+        slices = (
+            slice(localBox.minx - bounds.minx, localBox.maxx - bounds.minx),
+            slice(localBox.minz - bounds.minz, localBox.maxz - bounds.minz),
+            slice(localBox.miny - bounds.miny, localBox.maxy - bounds.miny),
+        )
+        return localBox, slices
+
+
 class FakeChunk(ChunkBase):
     @property
     def HeightMap(self):

test/anvil_test.py

@@ -9,6 +9,7 @@ from infiniteworld import MCInfdevOldLevel
 import nbt
 from schematic import MCSchematic
 from box import BoundingBox
+import block_copy
 from templevel import mktemp, TempLevel
 
 __author__ = 'Rio'
@@ -50,7 +51,7 @@ class TestAnvilLevel(unittest.TestCase):
         cx, cz = level.allChunks.next()
         level.copyBlocksFrom(indevlevel, BoundingBox((0, 0, 0), (256, 128, 256)), (cx * 16, 0, cz * 16))
 
-        convertedSourceBlocks, convertedSourceData = indevlevel.convertBlocksFromLevel(level, indevlevel.Blocks[0:16, 0:16, 0:indevlevel.Height], indevlevel.Data[0:16, 0:16, 0:indevlevel.Height])
+        convertedSourceBlocks, convertedSourceData = block_copy.convertBlocks(indevlevel, level, indevlevel.Blocks[0:16, 0:16, 0:indevlevel.Height], indevlevel.Data[0:16, 0:16, 0:indevlevel.Height])
         assert (level.getChunk(cx, cz).Blocks[0:16, 0:16, 0:indevlevel.Height] == convertedSourceBlocks).all()
 
     def testImportSchematic(self):
@@ -62,7 +63,7 @@ class TestAnvilLevel(unittest.TestCase):
         level.copyBlocksFrom(schem, schem.bounds, (0, 64, 0))
         schem = MCSchematic(shape=schem.bounds.size)
         schem.copyBlocksFrom(level, box, (0, 0, 0))
-        convertedSourceBlocks, convertedSourceData = schem.convertBlocksFromLevel(level, schem.Blocks, schem.Data)
+        convertedSourceBlocks, convertedSourceData = block_copy.convertBlocks(schem, level, schem.Blocks, schem.Data)
         assert (level.getChunk(cx, cz).Blocks[0:1, 0:3, 64:65] == convertedSourceBlocks).all()
 
     def testRecreateChunks(self):