AstralTransRocketries/pymclevel
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'renamedAllChunks'

Browse Source 
renamed presentChunks to loadedChunks for clarity, and allChunks will cause a chunk scan the first time it is called
This commit is contained in:
David Vierra 2010-11-29 22:38:03 -10:00
commit ef8432ee57
4 changed files with 155 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
box.py
View File

@ -91,7 +91,7 @@ class BoundingBox (object):
maximum = property(getMaximum, None, None, "The endpoint of the box; origin plus size.")
def getVolume(self): return reduce(int.__mul__, self.size)
def getVolume(self): return reduce(lambda a,b:a*b, self.size)
volume = property(getVolume, None, None, "The volume of the box in blocks")
@property

29
mce.py
View File

@ -438,9 +438,9 @@ class mce(object):
blockCounts = zeros( (256,), 'uint64')
sizeOnDisk = 0;
print "Analyzing {0} chunks...".format(len(self.level.presentChunks))
print "Analyzing {0} chunks...".format(len(self.level.allChunks))
for i, cPos in enumerate(self.level.presentChunks, 1):
for i, cPos in enumerate(self.level.allChunks, 1):
ch = self.level.getChunk(*cPos);
counts = bincount(ch.Blocks.ravel())
blockCounts[:counts.shape[0]] += counts
@ -585,7 +585,7 @@ class mce(object):
print "Dumping signs..."
signCount = 0;
for i, cPos in enumerate(self.level.presentChunks):
for i, cPos in enumerate(self.level.allChunks):
try:
chunk = self.level.getChunk(*cPos);
except mclevel.ChunkMalformed:
@ -649,7 +649,7 @@ class mce(object):
print "Removing all entities except Painting..."
def match(entityID): return entityID != "Painting";
for cx,cz in self.level.presentChunks:
for cx,cz in self.level.allChunks:
chunk = self.level.getChunk(cx,cz)
entitiesRemoved = 0;
@ -692,10 +692,9 @@ class mce(object):
box = self.readBox(command)
oldChunkCount = len(self.level.presentChunks)
self.level.createChunksInBox(box)
chunksCreated = self.level.createChunksInBox(box)
print "Created {0} chunks." .format(len(self.level.presentChunks)-oldChunkCount)
print "Created {0} chunks." .format(len(chunksCreated))
self.needsSave = True;
@ -712,10 +711,9 @@ class mce(object):
box = self.readBox(command)
oldChunkCount = len(self.level.presentChunks)
self.level.deleteChunksInBox(box)
deletedChunks = self.level.deleteChunksInBox(box)
print "Deleted {0} chunks." .format(oldChunkCount-len(self.level.presentChunks))
print "Deleted {0} chunks." .format(len(deletedChunks))
def _prune(self, command):
"""
@ -730,14 +728,13 @@ class mce(object):
box = self.readBox(command)
oldChunkCount = len(self.level.presentChunks)
for cx,cz in self.level.presentChunks:
i=0;
for cx,cz in self.level.allChunks:
if cx < box.mincx or cx >= box.maxcx or cz < box.mincz or cz >= box.maxcz:
self.level.deleteChunk(cx,cz)
i+=1;
print "Pruned {0} chunks." .format(oldChunkCount-len(self.level.presentChunks))
print "Pruned {0} chunks." .format(i)
def _relight(self, command):
"""
@ -751,7 +748,7 @@ class mce(object):
chunks = itertools.product(range(box.mincx, box.maxcx),range(box.mincz, box.maxcz))
else:
chunks = self.level.presentChunks
chunks = self.level.allChunks
self.level.generateLights(chunks)

183
mclevel.py
View File

@ -41,10 +41,13 @@ ourworld = mclevel.fromFile("C:\\Minecraft\\OurWorld");
# Convenience method to load a numbered world from the saves folder.
world1 = mclevel.loadWorldNumber(1);
# Find out which chunks are present
chunkPositions = world1.presentChunks
# Find out which chunks are present. Doing this will scan the chunk folders the
# first time it is used. If you already know where you want to be, skip to
# world1.getChunk(xPos, zPos)
# presentChunks returns a list of tuples (xPos, zPos)
chunkPositions = world1.allChunks
# allChunks returns a list of tuples (xPos, zPos)
xPos, zPos = chunkPositions[0];
# retrieve an InfdevChunk object. getChunk is a special method;
@ -315,7 +318,7 @@ def unpack_first(func):
class MCLevel(object):
""" MCLevel is an abstract class providing many routines to the different level types,
including a common copyEntitiesFrom built on class-specific routines, and
a dummy getChunk/getPresentChunks for the finite levels.
a dummy getChunk/allChunks for the finite levels.
MCLevel subclasses must have Width, Length, and Height attributes. The first two are always zero for infinite levels.
Subclasses must also have Blocks, and optionally Data and BlockLight.
@ -337,6 +340,7 @@ class MCLevel(object):
players = ["Player"]
dimNo = 0;
parentWorld = None
world = None
@classmethod
def isLevel(cls, filename):
"""Tries to find out whether the given filename can be loaded
@ -422,14 +426,14 @@ class MCLevel(object):
self.root_tag = nbt.load(buf=fromstring(data, dtype='uint8'));
except Exception, e:
error( u"Malformed NBT data in file: {0} ({1})".format(self.filename, e) )
self.world.malformedChunk(*self.chunkPosition);
if self.world: self.world.malformedChunk(*self.chunkPosition);
raise ChunkMalformed, self.filename
try:
self.shapeChunkData()
except KeyError:
error( u"Incorrect chunk format in file: " + self.filename )
self.world.malformedChunk(*self.chunkPosition);
if self.world: self.world.malformedChunk(*self.chunkPosition);
raise ChunkMalformed, self.filename
self.dataIsPacked = True;
@ -443,12 +447,16 @@ class MCLevel(object):
def addEntity(self, *args): pass
def addTileEntity(self, *args): pass
def loadChunk(self, cx, cz ):
pass;
@property
def loadedChunks(self):
return itertools.product(xrange(0, self.Width+15>>4), xrange(0, self.Length+15>>4))
@property
def presentChunks(self):
return itertools.product(xrange(0, self.Width+15>>4), xrange(0, self.Length+15>>4))
def presentChunks(self): return self.allChunks #backward compatibility
@property
def allChunks(self):
return self.loadedChunks
def getChunk(self, cx, cz):
#if not hasattr(self, 'whiteLight'):
@ -696,6 +704,9 @@ class MCLevel(object):
'''
info( u"Identifying " + filename )
class LoadingError(RuntimeError): pass
if not filename:
raise IOError, "File not found: "+filename
if not os.path.exists(filename):
@ -761,9 +772,12 @@ class MCLevel(object):
except Exception, e:
info( u"Error during NBT load: {0!r}".format(e) )
info( u"Fallback: Detected compressed flat block array, yzx ordered " )
try:
lev = MCJavaLevel(filename, data);
lev.compressed = compressed;
return lev;
except Exception, e2:
raise LoadingError, ("Multiple errors encountered", e, e2)
else:
if(MCIndevLevel._isTagLevel(root_tag)):
@ -1456,6 +1470,9 @@ class InfdevChunk(MCLevel):
if create:
self.create();
else:
if not world.containsChunk(*chunkPosition):
raise ChunkNotPresent("File not found: {0}", self.filename)
def compress(self):
@ -1804,14 +1821,14 @@ class MCInfdevOldLevel(MCLevel):
return False
def getWorldBounds(self):
if len(self.presentChunks) == 0:
if len(self.allChunks) == 0:
return BoundingBox( (0,0,0), (0,0,0) )
presentChunksArray = array(self.presentChunks)
mincx = min(presentChunksArray[:,0])
maxcx = max(presentChunksArray[:,0])
mincz = min(presentChunksArray[:,1])
maxcz = max(presentChunksArray[:,1])
allChunksArray = array(list(self.allChunks), dtype='int32')
mincx = min(allChunksArray[:,0])
maxcx = max(allChunksArray[:,0])
mincz = min(allChunksArray[:,1])
maxcz = max(allChunksArray[:,1])
origin = (mincx << 4, 0, mincz << 4)
size = ((maxcx-mincx+1) << 4, 128, (maxcz-mincz+1) << 4)
@ -1950,7 +1967,8 @@ class MCInfdevOldLevel(MCLevel):
self.filename = os.path.join(self.worldDir, "level.dat")
#maps (cx,cz) pairs to InfdevChunks
self._presentChunksDict = None;
self._loadedChunks = {}
self._allChunks = None
self.dimensions = {};
#used to limit memory usage
@ -2008,7 +2026,7 @@ class MCInfdevOldLevel(MCLevel):
def preloadChunkPaths(self):
info( u"Scanning for chunks..." )
worldDirs = os.listdir(self.worldDir);
self._presentChunksDict = {};
self._allChunks = set()
for dirname in worldDirs:
if(dirname in self.dirhashes):
@ -2030,25 +2048,20 @@ class MCInfdevOldLevel(MCLevel):
except Exception, e:
info( 'Skipped file {0} ({1})'.format('.'.join(c), e) )
continue
self._presentChunks[ (cx, cz) ] = InfdevChunk(self, (cx, cz));
info( u"Found {0} chunks.".format(len(self._presentChunks)) )
#self._presentChunks.update(dict(zip(chunks, fullpaths)));
## for filename, chunk in zip(fullpaths, chunks):
## chunkfh = file(filename, 'rb')
## self.compressedTags[chunk] = chunkfh.read();
## chunkfh.close();
self._allChunks.add( (cx,cz) )
#
info( u"Found {0} chunks.".format(len(self.allChunks)) )
def compressAllChunks(self):
for ch in self._presentChunks.itervalues():
for ch in self._loadedChunks.itervalues():
ch.compress();
def compressChunk(self, x, z):
if not (x,z) in self._presentChunks: return; #not an error
self._presentChunks[x,z].compress()
if not (x,z) in self._loadedChunks: return; #not an error
self._loadedChunks[x,z].compress()
decompressedChunkLimit = 2048 # about 320 megabytes
loadedChunkLimit = 8192 # from 8mb to 800mb depending on chunk contents
@ -2088,7 +2101,7 @@ class MCInfdevOldLevel(MCLevel):
def chunkFilenameAt(self, x, y, z):
cx = x >> 4
cz = z >> 4
return self._presentChunks.get( (cx, cz) ).filename
return self._loadedChunks.get( (cx, cz) ).filename
base36alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"
def decbase36(self, s):
@ -2253,48 +2266,61 @@ class MCInfdevOldLevel(MCLevel):
#the heightmap is ordered differently because in minecraft it is a flat array
@property
def presentChunks(self):
return self._presentChunks.keys();
def loadedChunks(self):
return self._loadedChunks.keys();
@property
def _presentChunks(self):
if self._presentChunksDict is None:
self.preloadChunkPaths();
return self._presentChunksDict
def allChunks(self):
if self._allChunks is None:
self.preloadChunkPaths()
return self._allChunks;
def getChunks(self, chunks = None):
""" pass a list of chunk coordinate tuples to get a list of InfdevChunks.
pass nothing for a list of every chunk in the level.
the chunks are automatically loaded."""
if chunks is None: chunks = self.presentChunks;
return [self.getChunk(cx,cz) for (cx,cz) in chunks if (cx,cz) in self._presentChunks]
if chunks is None: chunks = self.allChunks;
return [self.getChunk(cx,cz) for (cx,cz) in chunks if self.containsChunk(cx,cz)]
def _makeChunk(self, cx,cz):
"""return the chunk object at the given position, creating it if necessary.
because loading the chunk is done later, accesses to chunk attributes may
raise ChunkMalformed"""
if not self.containsChunk(cx,cz):
raise ChunkNotPresent, (cx,cz);
if not (cx,cz) in self._loadedChunks:
self._loadedChunks[cx,cz] = InfdevChunk(self, (cx, cz));
return self._loadedChunks[cx,cz]
def getChunk(self, cx, cz):
""" read the chunk from disk, load it, and return it.
decompression and unpacking is done lazily."""
if not (cx,cz) in self._presentChunks:
raise ChunkNotPresent, "Chunk {0} not present".format((cx,cz))
c = self._presentChunks[cx,cz]
c = self._makeChunk(cx,cz)
c.load();
if not (cx,cz) in self._presentChunks:
if not (cx,cz) in self._loadedChunks:
raise ChunkMalformed, "Chunk {0} malformed".format((cx,cz))
self.world.malformedChunk(*self.chunkPosition);
return c;
def markDirtyChunk(self, cx, cz):
if not (cx,cz) in self._presentChunks: return
self._presentChunks[cx,cz].chunkChanged();
if not (cx,cz) in self._loadedChunks: return
self._loadedChunks[cx,cz].chunkChanged();
def saveInPlace(self):
for level in self.dimensions.itervalues():
level.saveInPlace(True);
dirtyChunkCount = 0;
for chunk in self._presentChunks.itervalues():
if self._loadedChunks:
for chunk in self._loadedChunks.itervalues():
if chunk.dirty:
dirtyChunkCount += 1;
chunk.save();
@ -2311,11 +2337,12 @@ class MCInfdevOldLevel(MCLevel):
startTime = datetime.now();
if dirtyChunks is None:
dirtyChunks = filter(lambda x: x.needsLighting, self._presentChunks.values());
dirtyChunks = (ch for ch in self._loadedChunks.itervalues() if ch.needsLighting)
else:
dirtyChunks = [self._presentChunks[c] for c in dirtyChunks if c in self._presentChunks];
dirtyChunks = (self._makeChunk(*c) for c in dirtyChunks if self.containsChunk(*c))
dirtyChunks = sorted(dirtyChunks, key=lambda x:x.chunkPosition)
dirtyChunks = sorted(list(dirtyChunks), key=lambda x:x.chunkPosition)
#at 150k per loaded chunk,
maxLightingChunks = 4000
@ -2383,8 +2410,8 @@ class MCInfdevOldLevel(MCLevel):
#relight all blocks in neighboring chunks in case their light source disappeared.
cx,cz = ch.chunkPosition
for dx,dz in itertools.product( (-1, 0, 1), (-1, 0, 1) ):
if (cx+dx,cz+dz) in self._presentChunks:
dirtyChunks.add(self._presentChunks[(cx+dx,cz+dz)]);
if (cx+dx,cz+dz) in self._loadedChunks:
dirtyChunks.add(self._loadedChunks[(cx+dx,cz+dz)]);
dirtyChunks = sorted(dirtyChunks, key=lambda x:x.chunkPosition)
@ -2453,10 +2480,10 @@ class MCInfdevOldLevel(MCLevel):
print "Chunk error during relight, chunk skipped: ", e
continue;
for dir,dx,dy,dz in ( (FaceXDecreasing,-1,0,0),
(FaceXIncreasing,1,0,0),
(FaceZDecreasing,0,0, -1),
(FaceZIncreasing,0,0, 1) ):
for dir,dx,dz in ( (FaceXDecreasing,-1,0),
(FaceXIncreasing,1,0),
(FaceZDecreasing,0, -1),
(FaceZIncreasing,0, 1) ):
try:
neighboringChunks[dir] = self.getChunk(cx+dx,cz+dz)
except (ChunkNotPresent, ChunkMalformed):
@ -2719,7 +2746,7 @@ class MCInfdevOldLevel(MCLevel):
def getAllChunkSlices(self):
for cpos in self.presentChunks:
for cpos in self.allChunks:
xPos, zPos = cpos
try:
chunk = self.getChunk(xPos, zPos)
@ -2917,7 +2944,7 @@ class MCInfdevOldLevel(MCLevel):
for dx, dz in itertools.product( (-1, 0, 1), (-1, 0, 1) ):
ncPos = (cx+dx, cz+dz);
if ncPos not in changedChunkPositions:
ch = self._presentChunks.get((cx,cz), None);
ch = self._loadedChunks.get((cx,cz), None);
if ch:
ch.needsLighting = True
@ -2957,17 +2984,21 @@ class MCInfdevOldLevel(MCLevel):
return self.containsChunk(x>>4, z>>4)
def containsChunk(self, cx, cz):
return (cx, cz) in self._presentChunks;
if self._allChunks is not None: return (cx, cz) in self._allChunks;
if (cx,cz) in self._loadedChunks: return True;
return os.path.exists(self.chunkFilename(cx,cz))
def malformedChunk(self, cx, cz):
debug( u"Forgetting malformed chunk {0} ({1})".format((cx,cz), self.chunkFilename(cx,cz)) )
if (cx,cz) in self._presentChunks:
del self._presentChunks[(cx,cz)]
if (cx,cz) in self._loadedChunks:
del self._loadedChunks[(cx,cz)]
self._bounds = None
def createChunk(self, cx, cz):
if (cx,cz) in self._presentChunks: raise ValueError, "{0}:Chunk {1} already present!".format(self, (cx,cz) )
self._presentChunks[cx,cz] = InfdevChunk(self, (cx,cz), create = True)
if (cx,cz) in self._loadedChunks: raise ValueError, "{0}:Chunk {1} already present!".format(self, (cx,cz) )
if self._allChunks is not None: self._allChunks.add( (cx,cz) )
self._loadedChunks[cx,cz] = InfdevChunk(self, (cx,cz), create = True)
self._bounds = None
def createChunks(self, chunks):
@ -2976,7 +3007,7 @@ class MCInfdevOldLevel(MCLevel):
ret = [];
for cx,cz in chunks:
i+=1;
if not ((cx,cz) in self._presentChunks):
if not ((cx,cz) in self._loadedChunks):
ret.append( (cx,cz) )
self.createChunk(cx,cz);
self.compressChunk(cx,cz);
@ -2993,20 +3024,25 @@ class MCInfdevOldLevel(MCLevel):
return self.createChunks(box.chunkPositions);
def deleteChunk(self, cx, cz):
if not (cx,cz) in self._presentChunks: return;
self._presentChunks[(cx,cz)].remove();
if not (cx,cz) in self._loadedChunks: return;
self._loadedChunks[(cx,cz)].remove();
if self._allChunks is not None: self._allChunks.discard( (cx,cz) )
del self._presentChunks[(cx,cz)]
del self._loadedChunks[(cx,cz)]
self._bounds = None
def deleteChunksInBox(self, box):
info( u"Deleting {0} chunks in {1}".format((box.maxcx-box.mincx)*( box.maxcz-box.mincz), ((box.mincx, box.mincz), (box.maxcx, box.maxcz))) )
i=0;
ret = [];
for cx,cz in itertools.product(xrange(box.mincx,box.maxcx), xrange(box.mincz, box.maxcz)):
i+=1;
if ((cx,cz) in self._presentChunks):
if self.containsChunk(cx,cz):
self.deleteChunk(cx,cz);
ret.append( (cx,cz) )
assert not self.containsChunk(cx,cz), "Just deleted {0} but it didn't take".format((cx,cz))
if i%100 == 0:
info( u"Chunk {0}...".format( i ) )
@ -3137,7 +3173,8 @@ class ZipSchematic (MCInfdevOldLevel):
zf = ZipFile(filename)
self.zipfile = zf
self._presentChunksDict = None;
self._loadedChunks = {};
self._allChunks = None
self.dimensions = {};
self.loadLevelDat(False, 0, 0)
@ -3172,9 +3209,12 @@ class ZipSchematic (MCInfdevOldLevel):
def saveInPlace(self):
raise NotImplementedError, "Cannot save zipfiles yet!"
def containsChunk(self, cx, cz):
return (cx,cz) in self.allChunks
def preloadChunkPaths(self):
info( u"Scanning for chunks..." )
self._presentChunksDict = {}
self._allChunks = set()
infos = self.zipfile.infolist()
names = [i.filename.split('/') for i in infos]
@ -3188,9 +3228,10 @@ class ZipSchematic (MCInfdevOldLevel):
except Exception, e:
info( 'Skipped file {0} ({1})'.format('.'.join(c), e) )
continue
self._presentChunksDict[ (cx, cz) ] = InfdevChunk(self, (cx, cz));
#self._loadedChunks[ (cx, cz) ] = InfdevChunk(self, (cx, cz));
self._allChunks.add( (cx,cz) )
info( u"Found {0} chunks.".format(len(self._presentChunksDict)) )
info( u"Found {0} chunks.".format(len(self._allChunks)) )
def preloadDimensions(self):
@ -3605,7 +3646,7 @@ def testAlphaLevels():
indevlevel = MCLevel.fromFile("hell.mclevel")
level = MCInfdevOldLevel(filename="d:\Testworld");
for ch in level.presentChunks: level.deleteChunk(*ch)
for ch in level.allChunks: level.deleteChunk(*ch)
level.createChunksInBox( BoundingBox((0,0,0), (32, 0, 32)) )
level.copyBlocksFrom(indevlevel, BoundingBox((0,0,0), (256, 128, 256)), (-0, 0, 0))
assert all(level.getChunk(0,0).Blocks[0:16,0:16,0:indevlevel.Height] == indevlevel.conversionTableFromLevel(level)[indevlevel.Blocks[0:16,0:16,0:indevlevel.Height]]);

13
run_regression_test.py
View File

@ -169,6 +169,9 @@ def main(argv):
with untared_content("regression_test/alpha.tar.gz") as directory:
test_data = os.path.join(directory, "alpha")
passes = []
fails = []
for func, name, sha, args in alpha_tests:
if any(fnmatch.fnmatch(name, x) for x in do_these_regressions):
if options.profile:
@ -177,9 +180,15 @@ def main(argv):
try:
func(test_data, sha, args)
except RegressionError, e:
print "Regression {0} failed: {1}".format(name, e)
fails.append( "Regression {0} failed: {1}".format(name, e) )
print fails[-1]
else:
print "Regression {0!r} complete.".format(name)
passes.append( "Regression {0!r} complete.".format(name) )
print passes[-1]
print "{0} tests passed.".format(len(passes))
for line in fails: print line;
if __name__ == '__main__':
sys.exit(main(sys.argv))