Experiment with having copyBlocks issue lighting updates only for blocks whose opacity or brightness changed.
copyBlocks accepts more values for updateLights time_relight_manmade has another option to select the scheme of issuing updates Add some tests results, conclusions, and blabber to time_relight_manmade
This commit is contained in:
David Vierra
parent
12e0ff3a22
commit
1e4ae2e688
@ -9,60 +9,255 @@ from mceditlib.test import templevel
|
||||
from mceditlib import relight
|
||||
|
||||
|
||||
def manmade_relight():
|
||||
def do_copy(dim, station, relight):
|
||||
times = 1
|
||||
boxes = []
|
||||
for x in range(times):
|
||||
for z in range(times):
|
||||
origin = (x * station.bounds.width, 54, z * station.bounds.length)
|
||||
boxes.append(BoundingBox(origin, station.bounds.size))
|
||||
dim.copyBlocks(station, station.bounds, origin, create=True, updateLights=relight)
|
||||
return reduce(lambda a, b: a.union(b), boxes)
|
||||
|
||||
def manmade_relight(test):
|
||||
world = templevel.TempLevel("AnvilWorld")
|
||||
dim = world.getDimension()
|
||||
stationEditor = WorldEditor("test_files/station.schematic")
|
||||
station = stationEditor.getDimension()
|
||||
|
||||
times = 1
|
||||
boxes = []
|
||||
|
||||
for x in range(times):
|
||||
for z in range(times):
|
||||
origin = (x * station.bounds.width, 54, z * station.bounds.length)
|
||||
boxes.append(BoundingBox(origin, station.bounds.size))
|
||||
dim.copyBlocks(station, station.bounds, origin, create=True)
|
||||
|
||||
box = reduce(lambda a, b: a.union(b), boxes)
|
||||
startCopy = time.time()
|
||||
box = do_copy(dim, station, False)
|
||||
copyTime = time.time() - startCopy
|
||||
print("Copy took %f seconds. Reducing relight-in-copyBlocks times by this much." % copyTime)
|
||||
|
||||
positions = []
|
||||
for cx, cz in box.chunkPositions():
|
||||
for cy in box.sectionPositions(cx, cz):
|
||||
positions.append((cx, cy, cz))
|
||||
assert len(positions) > box.chunkCount
|
||||
|
||||
poses = iter(positions)
|
||||
if test == "post" or test == "all":
|
||||
def postCopy(): # profiling
|
||||
start = time.time()
|
||||
count = 0
|
||||
print("Relighting outside of copyBlocks. Updating %d cells" % (len(positions) * 16 * 16 * 16))
|
||||
for cx, cy, cz in positions:
|
||||
indices = numpy.indices((16, 16, 16), numpy.int32)
|
||||
indices.shape = 3, 16*16*16
|
||||
indices += ([cx << 4], [cy << 4], [cz << 4])
|
||||
x, y, z = indices
|
||||
relight.updateLightsByCoord(dim, x, y, z)
|
||||
count += 1
|
||||
t = time.time() - start
|
||||
|
||||
def do_relight():
|
||||
cx, cy, cz = poses.next()
|
||||
indices = numpy.indices((16, 16, 16), numpy.int32)
|
||||
indices.shape = 3, 16*16*16
|
||||
indices += ([cx << 4], [cy << 4], [cz << 4])
|
||||
x, y, z = indices
|
||||
print "Relight manmade building (outside copyBlocks): " \
|
||||
"%d (out of %d) chunk-sections in %.02f seconds (%f sections per second; %dms per section)" \
|
||||
% (count, len(positions), t, count / t, 1000 * t / count)
|
||||
postCopy()
|
||||
|
||||
relight.updateLightsByCoord(dim, x, y, z)
|
||||
if test == "smart" or test == "all":
|
||||
def allSections():
|
||||
world = templevel.TempLevel("AnvilWorld")
|
||||
dim = world.getDimension()
|
||||
|
||||
# Find out how many sections we can do in `maxtime` seconds.
|
||||
start = time.time()
|
||||
count = 0
|
||||
maxtime = 10
|
||||
end = start + maxtime
|
||||
while time.time() < end:
|
||||
try:
|
||||
do_relight()
|
||||
except StopIteration:
|
||||
break
|
||||
count += 1
|
||||
t = time.time() - start
|
||||
start = time.time()
|
||||
do_copy(dim, station, "all")
|
||||
t = time.time() - start - copyTime
|
||||
|
||||
print "Relight manmade building: %d (out of %d) chunk-sections in %.02f seconds (%f sections per second; %dms per section)" % (count, len(positions), t, count / t, 1000 * t / count)
|
||||
print "Relight manmade building (in copyBlocks, all sections): " \
|
||||
"%d chunk-sections in %.02f seconds (%f sections per second; %dms per section)" \
|
||||
% (len(positions), t, len(positions) / t, 1000 * t / len(positions))
|
||||
allSections()
|
||||
|
||||
if test == "section" or test == "all":
|
||||
def perSection():
|
||||
world = templevel.TempLevel("AnvilWorld")
|
||||
dim = world.getDimension()
|
||||
|
||||
start = time.time()
|
||||
do_copy(dim, station, "section")
|
||||
t = time.time() - start - copyTime
|
||||
|
||||
print "Relight manmade building (in copyBlocks, for each section): " \
|
||||
"%d chunk-sections in %.02f seconds (%f sections per second; %dms per section)" \
|
||||
% (len(positions), t, len(positions) / t, 1000 * t / len(positions))
|
||||
perSection()
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) > 1:
|
||||
method = sys.argv[1]
|
||||
print "Using method", method
|
||||
relight.setMethod(method)
|
||||
manmade_relight()
|
||||
if len(sys.argv) > 2:
|
||||
test = sys.argv[2]
|
||||
else:
|
||||
test = "all"
|
||||
manmade_relight(test)
|
||||
|
||||
|
||||
"""
|
||||
Conclusion:
|
||||
|
||||
Much time is spent in the "post" method which updates all cells in the selection box, calling
|
||||
updateLights on cells whose opacity values did not change. This is evidenced by the time spent in
|
||||
"drawLights", which must be called because updateLights doesn't know the previous block type in
|
||||
that cell.
|
||||
|
||||
copyBlocksFrom has been modified to find the cells whose lighting or opacity value did change,
|
||||
and passing only those cells to updateLights. This is more than twice as fast, and updating
|
||||
all changed cells at once is even faster, presumably because changes to following chunks will
|
||||
invalidate lighting data computed by previous chunks.
|
||||
|
||||
Because updateLights does not know what the previous cell's opacity values were (it does know the
|
||||
cell's current light value, so it can skip spreadLight if the new brightness didn't exceed that),
|
||||
clients of updateLights should take care to find only cells whose opacity values changed.
|
||||
|
||||
copyBlocksFrom stores all changed cell positions, which could lead to MemoryErrors for very large
|
||||
copies. Instead of storing all positions, it should periodically call updateLights whenever the
|
||||
position list exceeds a threshold. This "batch-update" method should be an acceptable compromise
|
||||
between updating for each section (suffering invalidation costs), and updating all sections
|
||||
at once after the copy (risking MemoryErrors and possibly paying additional chunk loading costs)
|
||||
|
||||
Updating lights for chunks whose neighbors have not been copied yet will cause wasted effort.
|
||||
It helps to describe this graphically. This is the current visitation order:
|
||||
|
||||
(area is 24x12, and 34 chunks have been copied so far)
|
||||
|
||||
************************
|
||||
**********..............
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
|
||||
'.' represents chunks that are yet to be copied.
|
||||
'*' represents chunks that have been copied.
|
||||
|
||||
If a batched lighting update is called at this point, these are the chunks that, when they are
|
||||
copied over later, will invalidate parts of the previous update:
|
||||
|
||||
************************
|
||||
**********--------------
|
||||
----------+.............
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
|
||||
'-' represents chunks that when edited will invalidate the previous lighting update applied
|
||||
to the '*' chunks. There are 24 such chunks.
|
||||
|
||||
'+' represents chunks that when edited will invalidate at most half of a previous chunk's
|
||||
update.
|
||||
|
||||
So let's say 24.5 chunks are invalidated later. Out of 34 chunks, that is not very good at all.
|
||||
|
||||
That number is roughly proportional to the width of the selection box.
|
||||
|
||||
The current visitation order is thus:
|
||||
|
||||
|
||||
1234567890abcdefghijklmn
|
||||
opqrstuvwx--------------
|
||||
----------+.............
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
|
||||
|
||||
A possibly improved visitation order:
|
||||
|
||||
12efghuvwx-.............
|
||||
43dcjits--+.............
|
||||
589bknor-...............
|
||||
670almpq-...............
|
||||
--------+...............
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
........................
|
||||
|
||||
13 full chunks and two half-chunks are invalidated, for a total of 15 chunks out of 34.
|
||||
|
||||
At least it's less than half.
|
||||
|
||||
This number is roughly proportional to the square root of the number of chunks copied so far.
|
||||
|
||||
The order of chunks visited by copyBlocksFrom is linear. When it calls updateLights for a chunk,
|
||||
the chunks adjacent to that chunk (and ahead of that chunk in the order) will have to redo part
|
||||
of this chunk's lighting for the current chunk when they are copied. To minimize wasted effort,
|
||||
a chunk order that resembles a space-filling curve such as a Hilbert curve may be
|
||||
applicable. The goal is to reduce the number of chunks who have neighbors yet to be copied at the
|
||||
time the batched update is performed.
|
||||
|
||||
Maybe we can do better. What if, instead of batch-updating ALL of the chunks copied so far,
|
||||
we only batch-update the ones we know won't be invalidated later?
|
||||
|
||||
The cells that need update are currently just tossed in a list. Instead, associate them with
|
||||
their chunk position. Keep track of which chunks we have copied, and how many of their
|
||||
eight neighbors have already been copied too. Only issue a batch update for chunks where all eight
|
||||
neighbors are copied. If we use the original visitation order, then for very large copies, we may
|
||||
reach the threshold before any neighbors have been copied. The new visitation order would avoid
|
||||
this as, for most chunks, it will visit all of a chunk's neighbors very soon after that chunk.
|
||||
|
||||
In fact, it may not be necessary to batch-update at all if we can update a chunk as soon as all its
|
||||
neighbors are ready.
|
||||
|
||||
Output:
|
||||
Using method cython
|
||||
INFO:mceditlib.block_copy:Copying 3103771 blocks from BoundingBox(origin=Vector(0, 0, 0), size=Vector(113, 121, 227)) to (0, 54, 0)
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 20/120...
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 40/120...
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 60/120...
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 80/120...
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 100/120...
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 120/120...
|
||||
INFO:mceditlib.block_copy:Duration: 1.292s, 120/120 chunks, 10.77ms per chunk (92.88 chunks per second)
|
||||
INFO:mceditlib.block_copy:Copied 0/0 entities and 293/293 tile entities
|
||||
Copy took 1.292000 seconds. Reducing relight-in-copyBlocks times by this much.
|
||||
Relighting outside of copyBlocks. Updating 3932160 cells
|
||||
Relight manmade building (outside copyBlocks): 960 (out of 960) chunk-sections in 71.49 seconds (13.428639 sections per second; 74ms per section)
|
||||
INFO:mceditlib.block_copy:Copying 3103771 blocks from BoundingBox(origin=Vector(0, 0, 0), size=Vector(113, 121, 227)) to (0, 54, 0)
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 20/120...
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 40/120...
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 60/120...
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 80/120...
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 100/120...
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 120/120...
|
||||
INFO:mceditlib.block_copy:Duration: 1.318s, 120/120 chunks, 10.98ms per chunk (91.05 chunks per second)
|
||||
INFO:mceditlib.block_copy:Copied 0/0 entities and 293/293 tile entities
|
||||
INFO:mceditlib.block_copy:Updating all at once for 969 sections (646338 cells)
|
||||
INFO:mceditlib.block_copy:Lighting complete.
|
||||
INFO:mceditlib.block_copy:Duration: 16.979s, 968 sections, 17.54ms per section (57.01 sections per second)
|
||||
Relight manmade building (in copyBlocks, all sections): 960 chunk-sections in 17.01 seconds (56.444027 sections per second; 17ms per section)
|
||||
INFO:mceditlib.block_copy:Copying 3103771 blocks from BoundingBox(origin=Vector(0, 0, 0), size=Vector(113, 121, 227)) to (0, 54, 0)
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 20/120...
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 40/120...
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 60/120...
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 80/120...
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 100/120...
|
||||
INFO:mceditlib.block_copy:Copying: Chunk 120/120...
|
||||
Relight manmade building (in copyBlocks, for each section): 960 chunk-sections in 26.12 seconds (36.757667 sections per second; 27ms per section)
|
||||
INFO:mceditlib.block_copy:Duration: 27.408s, 120/120 chunks, 228.40ms per chunk (4.38 chunks per second)
|
||||
INFO:mceditlib.block_copy:Copied 0/0 entities and 293/293 tile entities
|
||||
"""
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
"""
|
||||
import time
|
||||
import logging
|
||||
from mceditlib import relight
|
||||
from mceditlib.selection import BoundingBox, SectionBox
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@ -29,7 +30,7 @@ def sourceMaskFunc(blocksToCopy):
|
||||
return unmaskedSourceMask
|
||||
|
||||
|
||||
def copyBlocksIter(destDim, sourceDim, sourceSelection, destinationPoint, blocksToCopy=None, entities=True, create=False, biomes=False):
|
||||
def copyBlocksIter(destDim, sourceDim, sourceSelection, destinationPoint, blocksToCopy=None, entities=True, create=False, biomes=False, updateLights=False):
|
||||
"""
|
||||
Copy blocks and entities from the `sourceBox` area of `sourceDim` to `destDim` starting at `destinationPoint`.
|
||||
|
||||
@ -57,6 +58,11 @@ def copyBlocksIter(destDim, sourceDim, sourceSelection, destinationPoint, blocks
|
||||
entitiesSeen = 0
|
||||
tileEntitiesSeen = 0
|
||||
|
||||
if updateLights:
|
||||
allChangedX = []
|
||||
allChangedY = []
|
||||
allChangedZ = []
|
||||
|
||||
makeSourceMask = sourceMaskFunc(blocksToCopy)
|
||||
|
||||
copyOffset = destBox.origin - sourceSelection.origin
|
||||
@ -79,8 +85,8 @@ def copyBlocksIter(destDim, sourceDim, sourceSelection, destinationPoint, blocks
|
||||
|
||||
i += 1
|
||||
yield (i, chunkCount)
|
||||
if i % 100 == 0:
|
||||
log.info("Copying: Chunk {0}...".format(i))
|
||||
if i % 20 == 0:
|
||||
log.info("Copying: Chunk {0}/{1}...".format(i, chunkCount))
|
||||
|
||||
# Use sourceBiomeMask to accumulate a list of columns over all sections whose biomes should be copied.
|
||||
sourceBiomes = None
|
||||
@ -106,7 +112,7 @@ def copyBlocksIter(destDim, sourceDim, sourceSelection, destinationPoint, blocks
|
||||
sourceBiomeMask |= sourceMask.any(axis=0)
|
||||
|
||||
# Find corresponding destination area(s)
|
||||
sectionBox = SectionBox(sourceCpos[0], sourceCy, sourceCpos[1], sourceSection)
|
||||
sectionBox = SectionBox(sourceCpos[0], sourceCy, sourceCpos[1])
|
||||
destBox = BoundingBox(sectionBox.origin + copyOffset, sectionBox.size)
|
||||
|
||||
for destCpos in destBox.chunkPositions():
|
||||
@ -144,11 +150,46 @@ def copyBlocksIter(destDim, sourceDim, sourceSelection, destinationPoint, blocks
|
||||
|
||||
# Convert blocks
|
||||
convertedSourceBlocks, convertedSourceData = convertBlocks(sourceBlocks, sourceData)
|
||||
convertedSourceBlocksMasked = convertedSourceBlocks[sourceMaskPart]
|
||||
|
||||
# Find blocks that need direct lighting update - block opacity or brightness changed
|
||||
|
||||
oldBrightness = destDim.blocktypes.brightness[destSection.Blocks[destSlices][sourceMaskPart]]
|
||||
newBrightness = destDim.blocktypes.brightness[convertedSourceBlocksMasked]
|
||||
oldOpacity = destDim.blocktypes.opacity[destSection.Blocks[destSlices][sourceMaskPart]]
|
||||
newOpacity = destDim.blocktypes.opacity[convertedSourceBlocksMasked]
|
||||
changedLight = (oldBrightness != newBrightness) | (oldOpacity != newOpacity)
|
||||
|
||||
# Write blocks
|
||||
destSection.Blocks[destSlices][sourceMaskPart] = convertedSourceBlocks[sourceMaskPart]
|
||||
destSection.Data[destSlices][sourceMaskPart] = convertedSourceData[sourceMaskPart]
|
||||
|
||||
if updateLights:
|
||||
# Find coordinates of lighting updates
|
||||
(changedFlat,) = changedLight.nonzero()
|
||||
# Since convertedSourceBlocksMasked is a 1d array, changedFlat is an index
|
||||
# into this array. Thus, changedFlat is also an index into the nonzero values
|
||||
# of sourceMaskPart.
|
||||
|
||||
if len(changedFlat):
|
||||
x, y, z = sourceMaskPart.nonzero()
|
||||
changedX = x[changedFlat].astype('i4')
|
||||
changedY = y[changedFlat].astype('i4')
|
||||
changedZ = z[changedFlat].astype('i4')
|
||||
|
||||
changedX += intersect.minx
|
||||
changedY += intersect.miny
|
||||
changedZ += intersect.minz
|
||||
if updateLights == "all":
|
||||
allChangedX.append(changedX)
|
||||
allChangedY.append(changedY)
|
||||
allChangedZ.append(changedZ)
|
||||
else:
|
||||
# log.info("Updating section lights in %s blocks... (ob %s)",
|
||||
# changedFlat.shape,
|
||||
# oldBrightness.shape)
|
||||
relight.updateLightsByCoord(destDim, changedX, changedY, changedZ)
|
||||
|
||||
destChunk.dirty = True
|
||||
|
||||
# Copy biomes
|
||||
@ -176,10 +217,27 @@ def copyBlocksIter(destDim, sourceDim, sourceSelection, destinationPoint, blocks
|
||||
|
||||
duration = time.time() - startTime
|
||||
log.info("Duration: %0.3fs, %d/%d chunks, %0.2fms per chunk (%0.2f chunks per second)",
|
||||
duration, i, sourceSelection.chunkCount, 1000 * duration/i, i/duration)
|
||||
log.info("Copied %d/%d entities and %d/%d tile entities", entitiesCopied, entitiesSeen, tileEntitiesCopied, tileEntitiesSeen)
|
||||
|
||||
|
||||
|
||||
duration, i, sourceSelection.chunkCount, 1000 * duration/i, i/duration)
|
||||
log.info("Copied %d/%d entities and %d/%d tile entities",
|
||||
entitiesCopied, entitiesSeen, tileEntitiesCopied, tileEntitiesSeen)
|
||||
|
||||
if updateLights == "all":
|
||||
log.info("Updating all at once for %d sections (%d cells)", len(allChangedX), sum(len(a) for a in allChangedX))
|
||||
|
||||
startTime = time.time()
|
||||
|
||||
for i in range(len(allChangedX)):
|
||||
x = allChangedX[i]
|
||||
y = allChangedY[i]
|
||||
z = allChangedZ[i]
|
||||
relight.updateLightsByCoord(destDim, x, y, z)
|
||||
|
||||
i = i or 1
|
||||
duration = time.time() - startTime
|
||||
duration = duration or 1
|
||||
|
||||
log.info("Lighting complete.")
|
||||
log.info("Duration: %0.3fs, %d sections, %0.2fms per section (%0.2f sections per second)",
|
||||
duration, i, 1000 * duration/i, i/duration)
|
||||
|
||||
|
||||
|
||||
@ -739,14 +739,14 @@ class WorldEditorDimension(object):
|
||||
|
||||
# --- Import/Export ---
|
||||
|
||||
def copyBlocksIter(self, sourceLevel, sourceSelection, destinationPoint, blocksToCopy=None, entities=True, create=False, biomes=False):
|
||||
def copyBlocksIter(self, sourceLevel, sourceSelection, destinationPoint, blocksToCopy=None, entities=True, create=False, biomes=False, updateLights=False):
|
||||
return copyBlocksIter(self, sourceLevel, sourceSelection, destinationPoint, blocksToCopy, entities, create,
|
||||
biomes)
|
||||
biomes, updateLights)
|
||||
|
||||
|
||||
def copyBlocks(self, sourceLevel, sourceSelection, destinationPoint, blocksToCopy=None, entities=True, create=False, biomes=False):
|
||||
def copyBlocks(self, sourceLevel, sourceSelection, destinationPoint, blocksToCopy=None, entities=True, create=False, biomes=False, updateLights=False):
|
||||
return exhaust(self.copyBlocksIter(sourceLevel, sourceSelection, destinationPoint, blocksToCopy,
|
||||
entities, create, biomes))
|
||||
entities, create, biomes, updateLights))
|
||||
|
||||
def exportSchematicIter(self, selection):
|
||||
schematic = createSchematic(shape=selection.size, blocktypes=self.blocktypes)
|
||||
|
||||
Reference in New Issue
Block a user