From bb7e3392ce1e2b977d5025fb962863baed446f54 Mon Sep 17 00:00:00 2001 From: David Vierra Date: Sun, 21 Nov 2010 02:44:21 -1000 Subject: [PATCH 01/18] renamed presentChunks to allChunks, added loadedChunks --- mce.py | 29 +++++----- mclevel.py | 162 ++++++++++++++++++++++++++++++----------------------- 2 files changed, 104 insertions(+), 87 deletions(-) diff --git a/mce.py b/mce.py index d908b4d..63d38fb 100755 --- a/mce.py +++ b/mce.py @@ -438,9 +438,9 @@ class mce(object): blockCounts = zeros( (256,), 'uint64') sizeOnDisk = 0; - print "Analyzing {0} chunks...".format(len(self.level.presentChunks)) + print "Analyzing {0} chunks...".format(len(self.level.allChunks)) - for i, cPos in enumerate(self.level.presentChunks, 1): + for i, cPos in enumerate(self.level.allChunks, 1): ch = self.level.getChunk(*cPos); counts = bincount(ch.Blocks.ravel()) blockCounts[:counts.shape[0]] += counts @@ -585,7 +585,7 @@ class mce(object): print "Dumping signs..." signCount = 0; - for i, cPos in enumerate(self.level.presentChunks): + for i, cPos in enumerate(self.level.allChunks): try: chunk = self.level.getChunk(*cPos); except mclevel.ChunkMalformed: @@ -649,7 +649,7 @@ class mce(object): print "Removing all entities except Painting..." def match(entityID): return entityID != "Painting"; - for cx,cz in self.level.presentChunks: + for cx,cz in self.level.allChunks: chunk = self.level.getChunk(cx,cz) entitiesRemoved = 0; @@ -692,10 +692,9 @@ class mce(object): box = self.readBox(command) - oldChunkCount = len(self.level.presentChunks) - self.level.createChunksInBox(box) + chunksCreated = self.level.createChunksInBox(box) - print "Created {0} chunks." .format(len(self.level.presentChunks)-oldChunkCount) + print "Created {0} chunks." .format(len(chunksCreated)) self.needsSave = True; @@ -712,10 +711,9 @@ class mce(object): box = self.readBox(command) - oldChunkCount = len(self.level.presentChunks) - self.level.deleteChunksInBox(box) + deletedChunks = self.level.deleteChunksInBox(box) - print "Deleted {0} chunks." .format(oldChunkCount-len(self.level.presentChunks)) + print "Deleted {0} chunks." .format(len(deletedChunks)) def _prune(self, command): """ @@ -730,14 +728,13 @@ class mce(object): box = self.readBox(command) - oldChunkCount = len(self.level.presentChunks) - - - for cx,cz in self.level.presentChunks: + i=0; + for cx,cz in self.level.allChunks: if cx < box.mincx or cx >= box.maxcx or cz < box.mincz or cz >= box.maxcz: self.level.deleteChunk(cx,cz) + i+=1; - print "Pruned {0} chunks." .format(oldChunkCount-len(self.level.presentChunks)) + print "Pruned {0} chunks." .format(i) def _relight(self, command): """ @@ -751,7 +748,7 @@ class mce(object): chunks = itertools.product(range(box.mincx, box.maxcx),range(box.mincz, box.maxcz)) else: - chunks = self.level.presentChunks + chunks = self.level.allChunks self.level.generateLights(chunks) diff --git a/mclevel.py b/mclevel.py index 0bb9706..04eb453 100644 --- a/mclevel.py +++ b/mclevel.py @@ -41,10 +41,13 @@ ourworld = mclevel.fromFile("C:\\Minecraft\\OurWorld"); # Convenience method to load a numbered world from the saves folder. world1 = mclevel.loadWorldNumber(1); -# Find out which chunks are present -chunkPositions = world1.presentChunks +# Find out which chunks are present. Doing this will scan the chunk folders the +# first time it is used. If you already know where you want to be, skip to +# world1.getChunk(xPos, zPos) -# presentChunks returns a list of tuples (xPos, zPos) +chunkPositions = world1.allChunks + +# allChunks returns a list of tuples (xPos, zPos) xPos, zPos = chunkPositions[0]; # retrieve an InfdevChunk object. getChunk is a special method; @@ -442,18 +445,19 @@ class MCLevel(object): return None def addEntity(self, *args): pass def addTileEntity(self, *args): pass - - def loadChunk(self, cx, cz ): - pass; - + @property - def presentChunks(self): + def loadedChunks(self): return itertools.product(xrange(0, self.Width+15>>4), xrange(0, self.Length+15>>4)) + @property + def allChunks(self): + return self.loadedChunks + def getChunk(self, cx, cz): #if not hasattr(self, 'whiteLight'): #self.whiteLight = array([[[15] * self.Height] * 16] * 16, uint8); - + class FakeChunk: def load(self):pass def compress(self):pass @@ -1455,6 +1459,9 @@ class InfdevChunk(MCLevel): if create: self.create(); + else: + if not os.path.exists(self.filename): + raise ChunkNotPresent("File not found: {0}", self.filename) def compress(self): @@ -1803,14 +1810,14 @@ class MCInfdevOldLevel(MCLevel): return False def getWorldBounds(self): - if len(self.presentChunks) == 0: + if len(self.allChunks) == 0: return BoundingBox( (0,0,0), (0,0,0) ) - presentChunksArray = array(self.presentChunks) - mincx = min(presentChunksArray[:,0]) - maxcx = max(presentChunksArray[:,0]) - mincz = min(presentChunksArray[:,1]) - maxcz = max(presentChunksArray[:,1]) + allChunksArray = array(self.allChunks) + mincx = min(allChunksArray[:,0]) + maxcx = max(allChunksArray[:,0]) + mincz = min(allChunksArray[:,1]) + maxcz = max(allChunksArray[:,1]) origin = (mincx << 4, 0, mincz << 4) size = ((maxcx-mincx+1) << 4, 128, (maxcz-mincz+1) << 4) @@ -1949,7 +1956,8 @@ class MCInfdevOldLevel(MCLevel): self.filename = os.path.join(self.worldDir, "level.dat") #maps (cx,cz) pairs to InfdevChunks - self._presentChunksDict = None; + self._loadedChunks = {} + self._allChunks = None self.dimensions = {}; #used to limit memory usage @@ -2007,7 +2015,7 @@ class MCInfdevOldLevel(MCLevel): def preloadChunkPaths(self): info( u"Scanning for chunks..." ) worldDirs = os.listdir(self.worldDir); - self._presentChunksDict = {}; + self._allChunks = set() for dirname in worldDirs: if(dirname in self.dirhashes): @@ -2029,25 +2037,20 @@ class MCInfdevOldLevel(MCLevel): except Exception, e: info( 'Skipped file {0} ({1})'.format('.'.join(c), e) ) continue - self._presentChunks[ (cx, cz) ] = InfdevChunk(self, (cx, cz)); - info( u"Found {0} chunks.".format(len(self._presentChunks)) ) - - #self._presentChunks.update(dict(zip(chunks, fullpaths))); -## for filename, chunk in zip(fullpaths, chunks): -## chunkfh = file(filename, 'rb') -## self.compressedTags[chunk] = chunkfh.read(); -## chunkfh.close(); + self._allChunks.add( (cx,cz) ) + # + + info( u"Found {0} chunks.".format(len(self.allChunks)) ) - def compressAllChunks(self): - for ch in self._presentChunks.itervalues(): + for ch in self._loadedChunks.itervalues(): ch.compress(); def compressChunk(self, x, z): - if not (x,z) in self._presentChunks: return; #not an error - self._presentChunks[x,z].compress() + if not (x,z) in self._loadedChunks: return; #not an error + self._loadedChunks[x,z].compress() decompressedChunkLimit = 2048 # about 320 megabytes loadedChunkLimit = 8192 # from 8mb to 800mb depending on chunk contents @@ -2087,7 +2090,7 @@ class MCInfdevOldLevel(MCLevel): def chunkFilenameAt(self, x, y, z): cx = x >> 4 cz = z >> 4 - return self._presentChunks.get( (cx, cz) ).filename + return self._loadedChunks.get( (cx, cz) ).filename base36alphabet = "0123456789abcdefghijklmnopqrstuvwxyz" def decbase36(self, s): @@ -2252,51 +2255,60 @@ class MCInfdevOldLevel(MCLevel): #the heightmap is ordered differently because in minecraft it is a flat array @property - def presentChunks(self): - return self._presentChunks.keys(); - + def loadedChunks(self): + return self._loadedChunks.keys(); + @property - def _presentChunks(self): - if self._presentChunksDict is None: - self.preloadChunkPaths(); - return self._presentChunksDict + def allChunks(self): + if self._allChunks is None: + self.preloadChunkPaths() + return self._allChunks; + def getChunks(self, chunks = None): """ pass a list of chunk coordinate tuples to get a list of InfdevChunks. pass nothing for a list of every chunk in the level. the chunks are automatically loaded.""" - if chunks is None: chunks = self.presentChunks; - return [self.getChunk(cx,cz) for (cx,cz) in chunks if (cx,cz) in self._presentChunks] + if chunks is None: chunks = self.allChunks; + return [self.getChunk(cx,cz) for (cx,cz) in chunks if self.containsChunk(cx,cz)] def getChunk(self, cx, cz): """ read the chunk from disk, load it, and return it. decompression and unpacking is done lazily.""" + if self._allChunks is not None: + if not (cx,cz) in self._allChunks: + return; + - if not (cx,cz) in self._presentChunks: - raise ChunkNotPresent, "Chunk {0} not present".format((cx,cz)) - c = self._presentChunks[cx,cz] + if not (cx,cz) in self._loadedChunks: + self._loadedChunks[cx,cz] = InfdevChunk(self, (cx, cz)); + + + #raise ChunkNotPresent, "Chunk {0} not present".format((cx,cz)) + c = self._loadedChunks[cx,cz] c.load(); - if not (cx,cz) in self._presentChunks: + if not (cx,cz) in self._loadedChunks: raise ChunkMalformed, "Chunk {0} malformed".format((cx,cz)) self.world.malformedChunk(*self.chunkPosition); return c; def markDirtyChunk(self, cx, cz): - if not (cx,cz) in self._presentChunks: return - self._presentChunks[cx,cz].chunkChanged(); + if not (cx,cz) in self._loadedChunks: return + self._loadedChunks[cx,cz].chunkChanged(); def saveInPlace(self): for level in self.dimensions.itervalues(): level.saveInPlace(True); dirtyChunkCount = 0; - for chunk in self._presentChunks.itervalues(): - if chunk.dirty: - dirtyChunkCount += 1; - chunk.save(); + if self._loadedChunks: + for chunk in self._loadedChunks.itervalues(): + if chunk.dirty: + dirtyChunkCount += 1; + chunk.save(); self.root_tag.save(self.filename); @@ -2310,9 +2322,9 @@ class MCInfdevOldLevel(MCLevel): startTime = datetime.now(); if dirtyChunks is None: - dirtyChunks = filter(lambda x: x.needsLighting, self._presentChunks.values()); + dirtyChunks = filter(lambda x: x.needsLighting, self._loadedChunks.values()); else: - dirtyChunks = [self._presentChunks[c] for c in dirtyChunks if c in self._presentChunks]; + dirtyChunks = [self._loadedChunks[c] for c in dirtyChunks if c in self._loadedChunks]; dirtyChunks = sorted(list(dirtyChunks), key=lambda x:x.chunkPosition) @@ -2382,8 +2394,8 @@ class MCInfdevOldLevel(MCLevel): #relight all blocks in neighboring chunks in case their light source disappeared. cx,cz = ch.chunkPosition for dx,dz in itertools.product( (-1, 0, 1), (-1, 0, 1) ): - if (cx+dx,cz+dz) in self._presentChunks: - dirtyChunks.add(self._presentChunks[(cx+dx,cz+dz)]); + if (cx+dx,cz+dz) in self._loadedChunks: + dirtyChunks.add(self._loadedChunks[(cx+dx,cz+dz)]); dirtyChunks = sorted(dirtyChunks, key=lambda x:x.chunkPosition) @@ -2718,7 +2730,7 @@ class MCInfdevOldLevel(MCLevel): def getAllChunkSlices(self): - for cpos in self.presentChunks: + for cpos in self.allChunks: xPos, zPos = cpos try: chunk = self.getChunk(xPos, zPos) @@ -2912,7 +2924,7 @@ class MCInfdevOldLevel(MCLevel): for dx, dz in itertools.product( (-1, 0, 1), (-1, 0, 1) ): ncPos = (cx+dx, cz+dz); if ncPos not in changedChunkPositions: - ch = self._presentChunks.get((cx,cz), None); + ch = self._loadedChunks.get((cx,cz), None); if ch: ch.needsLighting = True @@ -2952,17 +2964,19 @@ class MCInfdevOldLevel(MCLevel): return self.containsChunk(x>>4, z>>4) def containsChunk(self, cx, cz): - return (cx, cz) in self._presentChunks; + if self._allChunks is not None: return (cx, cz) in self._allChunks; + if (cx,cz) in self._loadedChunks: return True; + return os.path.exists(self.chunkFilename(cx,cz)) def malformedChunk(self, cx, cz): debug( u"Forgetting malformed chunk {0} ({1})".format((cx,cz), self.chunkFilename(cx,cz)) ) - if (cx,cz) in self._presentChunks: - del self._presentChunks[(cx,cz)] + if (cx,cz) in self._loadedChunks: + del self._loadedChunks[(cx,cz)] self._bounds = None def createChunk(self, cx, cz): - if (cx,cz) in self._presentChunks: raise ValueError, "{0}:Chunk {1} already present!".format(self, (cx,cz) ) - self._presentChunks[cx,cz] = InfdevChunk(self, (cx,cz), create = True) + if (cx,cz) in self._loadedChunks: raise ValueError, "{0}:Chunk {1} already present!".format(self, (cx,cz) ) + self._loadedChunks[cx,cz] = InfdevChunk(self, (cx,cz), create = True) self._bounds = None def createChunks(self, chunks): @@ -2971,7 +2985,7 @@ class MCInfdevOldLevel(MCLevel): ret = []; for cx,cz in chunks: i+=1; - if not ((cx,cz) in self._presentChunks): + if not ((cx,cz) in self._loadedChunks): ret.append( (cx,cz) ) self.createChunk(cx,cz); self.compressChunk(cx,cz); @@ -2988,20 +3002,24 @@ class MCInfdevOldLevel(MCLevel): return self.createChunks(box.chunkPositions); def deleteChunk(self, cx, cz): - if not (cx,cz) in self._presentChunks: return; - self._presentChunks[(cx,cz)].remove(); + if not (cx,cz) in self._loadedChunks: return; + self._loadedChunks[(cx,cz)].remove(); - del self._presentChunks[(cx,cz)] + del self._loadedChunks[(cx,cz)] self._bounds = None def deleteChunksInBox(self, box): info( u"Deleting {0} chunks in {1}".format((box.maxcx-box.mincx)*( box.maxcz-box.mincz), ((box.mincx, box.mincz), (box.maxcx, box.maxcz))) ) i=0; + ret = []; for cx,cz in itertools.product(xrange(box.mincx,box.maxcx), xrange(box.mincz, box.maxcz)): i+=1; - if ((cx,cz) in self._presentChunks): + if self.containsChunk(cx,cz): self.deleteChunk(cx,cz); + ret.append( (cx,cz) ) + assert not self.containsChunk(cx,cz), "Just deleted {0} but it didn't take".format((cx,cz)) + if i%100 == 0: info( u"Chunk {0}...".format( i ) ) @@ -3132,7 +3150,8 @@ class ZipSchematic (MCInfdevOldLevel): zf = ZipFile(filename) self.zipfile = zf - self._presentChunksDict = None; + self._loadedChunks = {}; + self._allChunks = None self.dimensions = {}; self.loadLevelDat(False, 0, 0) @@ -3169,7 +3188,7 @@ class ZipSchematic (MCInfdevOldLevel): def preloadChunkPaths(self): info( u"Scanning for chunks..." ) - self._presentChunksDict = {} + self._allChunks = {} infos = self.zipfile.infolist() names = [i.filename.split('/') for i in infos] @@ -3183,9 +3202,10 @@ class ZipSchematic (MCInfdevOldLevel): except Exception, e: info( 'Skipped file {0} ({1})'.format('.'.join(c), e) ) continue - self._presentChunksDict[ (cx, cz) ] = InfdevChunk(self, (cx, cz)); - - info( u"Found {0} chunks.".format(len(self._presentChunksDict)) ) + #self._loadedChunks[ (cx, cz) ] = InfdevChunk(self, (cx, cz)); + self._allChunks.add( (cx,cz) ) + + info( u"Found {0} chunks.".format(len(self._allChunks)) ) def preloadDimensions(self): @@ -3600,7 +3620,7 @@ def testAlphaLevels(): indevlevel = MCLevel.fromFile("hell.mclevel") level = MCInfdevOldLevel(filename="d:\Testworld"); - for ch in level.presentChunks: level.deleteChunk(*ch) + for ch in level.allChunks: level.deleteChunk(*ch) level.createChunksInBox( BoundingBox((0,0,0), (32, 0, 32)) ) level.copyBlocksFrom(indevlevel, BoundingBox((0,0,0), (256, 128, 256)), (-0, 0, 0)) assert all(level.getChunk(0,0).Blocks[0:16,0:16,0:indevlevel.Height] == indevlevel.conversionTableFromLevel(level)[indevlevel.Blocks[0:16,0:16,0:indevlevel.Height]]); From 93466cc42a41763f09aba59ac44b38f7cecaefb0 Mon Sep 17 00:00:00 2001 From: David Vierra Date: Fri, 26 Nov 2010 03:25:36 -1000 Subject: [PATCH 02/18] presentChunks is now allChunks --- mclevel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mclevel.py b/mclevel.py index 0eb0c05..cbecd34 100644 --- a/mclevel.py +++ b/mclevel.py @@ -318,7 +318,7 @@ def unpack_first(func): class MCLevel(object): """ MCLevel is an abstract class providing many routines to the different level types, including a common copyEntitiesFrom built on class-specific routines, and - a dummy getChunk/getPresentChunks for the finite levels. + a dummy getChunk/allChunks for the finite levels. MCLevel subclasses must have Width, Length, and Height attributes. The first two are always zero for infinite levels. Subclasses must also have Blocks, and optionally Data and BlockLight. From 8f1724b0db48a2644ca861f963659054e3947677 Mon Sep 17 00:00:00 2001 From: David Vierra Date: Fri, 26 Nov 2010 03:42:47 -1000 Subject: [PATCH 03/18] need to make allChunks into a list before putting it into ndarray --- mclevel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mclevel.py b/mclevel.py index cbecd34..1daa9fe 100644 --- a/mclevel.py +++ b/mclevel.py @@ -1814,7 +1814,7 @@ class MCInfdevOldLevel(MCLevel): if len(self.allChunks) == 0: return BoundingBox( (0,0,0), (0,0,0) ) - allChunksArray = array(self.allChunks) + allChunksArray = array(list(self.allChunks), dtype='uint32') mincx = min(allChunksArray[:,0]) maxcx = max(allChunksArray[:,0]) mincz = min(allChunksArray[:,1]) From 767f3a9295044687bc18c6ccf80c288d5bf9b173 Mon Sep 17 00:00:00 2001 From: David Vierra Date: Fri, 26 Nov 2010 03:43:01 -1000 Subject: [PATCH 04/18] need to raise ChunkNotPresent instead of returning None --- mclevel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mclevel.py b/mclevel.py index 1daa9fe..fe8b3d7 100644 --- a/mclevel.py +++ b/mclevel.py @@ -2280,7 +2280,7 @@ class MCInfdevOldLevel(MCLevel): if self._allChunks is not None: if not (cx,cz) in self._allChunks: - return; + raise ChunkNotPresent, (cx,cz); if not (cx,cz) in self._loadedChunks: From 212d606cb2f9d516b2b554fd7b13af6df6c59e6b Mon Sep 17 00:00:00 2001 From: David Vierra Date: Fri, 26 Nov 2010 06:03:56 -1000 Subject: [PATCH 05/18] keep presentChunks property for compatibility, just calls allChunks --- mclevel.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mclevel.py b/mclevel.py index fe8b3d7..eae13e7 100644 --- a/mclevel.py +++ b/mclevel.py @@ -450,6 +450,9 @@ class MCLevel(object): def loadedChunks(self): return itertools.product(xrange(0, self.Width+15>>4), xrange(0, self.Length+15>>4)) + @property + def presentChunks(self): return self.allChunks #backward compatibility + @property def allChunks(self): return self.loadedChunks From ec8eeb503313532b8aaff36ea98f5ce5681b5375 Mon Sep 17 00:00:00 2001 From: David Vierra Date: Fri, 26 Nov 2010 18:48:11 -1000 Subject: [PATCH 06/18] changed launch_subprocess to use the existing environ --- run_regression_test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/run_regression_test.py b/run_regression_test.py index 054f15a..1c45270 100755 --- a/run_regression_test.py +++ b/run_regression_test.py @@ -72,9 +72,15 @@ def untared_content(src): yield dest def launch_subprocess(directory, arguments, env = {}): + #my python breaks with an empty environ, i think it wants PATH + #if sys.platform == "win32": + newenv = {} + newenv.update(os.environ) + newenv.update(env); + proc = subprocess.Popen((["python.exe"] if sys.platform == "win32" else []) + [ "./mce.py", - directory] + arguments, stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=env) + directory] + arguments, stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=newenv) return proc From f8a63b4eed66cb15dcf00b737ea317308c93e9c1 Mon Sep 17 00:00:00 2001 From: David Vierra Date: Fri, 26 Nov 2010 18:48:29 -1000 Subject: [PATCH 07/18] updated regression checksum. analyze now outputs the size on disk --- run_regression_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_regression_test.py b/run_regression_test.py index 1c45270..91c7592 100755 --- a/run_regression_test.py +++ b/run_regression_test.py @@ -148,7 +148,7 @@ def do_test_match_output(test_data, result_check, arguments=[]): alpha_tests = [ (do_test, 'baseline', '9e7460d39c8e0456789cf89fee45276db2719aaa', []), (do_test, 'degrief', '403e6c6147cf1f8d73377b18bbf5e4973606a311', ['degrief']), - (do_test_match_output, 'analyze', '5fc0fb54bacc9c6dfcafd5c6a48cc86b7dd5fc4b', ['analyze']), + (do_test_match_output, 'analyze', '1437f30df7e32c0a3708e264eda2d202c5efbfba', ['analyze']), (do_test, 'relight', 'e0cf60c62adfdb313f198af5314c31f89d158c12', ['relight']), (do_test, 'replace', 'd73767293e903b6d1c49c1838eb1849b69d83ad8', ['replace', 'Water', 'with', 'Lava']), (do_test, 'fill', 'f4f57c3d902b6894031d416cb9279232e7e24bd7', ['fill', 'Water']), From 81d9d0396f85a09c32c729969da6130a05fe9dc9 Mon Sep 17 00:00:00 2001 From: David Vierra Date: Sat, 27 Nov 2010 01:58:15 -1000 Subject: [PATCH 08/18] _allChunks needs to be a set! --- mclevel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mclevel.py b/mclevel.py index eae13e7..248c1e9 100644 --- a/mclevel.py +++ b/mclevel.py @@ -3196,7 +3196,7 @@ class ZipSchematic (MCInfdevOldLevel): def preloadChunkPaths(self): info( u"Scanning for chunks..." ) - self._allChunks = {} + self._allChunks = set() infos = self.zipfile.infolist() names = [i.filename.split('/') for i in infos] From beb4f6a62f5831d5bb944fd7605afdcff0c7abf1 Mon Sep 17 00:00:00 2001 From: David Vierra Date: Sat, 27 Nov 2010 02:39:26 -1000 Subject: [PATCH 09/18] when creating or deleting a chunk, update _allChunks if present --- mclevel.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mclevel.py b/mclevel.py index 248c1e9..b8eb396 100644 --- a/mclevel.py +++ b/mclevel.py @@ -2984,6 +2984,8 @@ class MCInfdevOldLevel(MCLevel): def createChunk(self, cx, cz): if (cx,cz) in self._loadedChunks: raise ValueError, "{0}:Chunk {1} already present!".format(self, (cx,cz) ) + if self._allChunks is not None: self._allChunks.add( (cx,cz) ) + self._loadedChunks[cx,cz] = InfdevChunk(self, (cx,cz), create = True) self._bounds = None @@ -3012,6 +3014,7 @@ class MCInfdevOldLevel(MCLevel): def deleteChunk(self, cx, cz): if not (cx,cz) in self._loadedChunks: return; self._loadedChunks[(cx,cz)].remove(); + if self._allChunks is not None: self._allChunks.discard( (cx,cz) ) del self._loadedChunks[(cx,cz)] self._bounds = None From 29119f7d77c8025623581774f6719db1c1e9bf2d Mon Sep 17 00:00:00 2001 From: David Vierra Date: Mon, 29 Nov 2010 05:49:08 -1000 Subject: [PATCH 10/18] allChunksArray contains chunk positions and needs to be signed! --- mclevel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mclevel.py b/mclevel.py index b8eb396..5fbecb2 100644 --- a/mclevel.py +++ b/mclevel.py @@ -1817,7 +1817,7 @@ class MCInfdevOldLevel(MCLevel): if len(self.allChunks) == 0: return BoundingBox( (0,0,0), (0,0,0) ) - allChunksArray = array(list(self.allChunks), dtype='uint32') + allChunksArray = array(list(self.allChunks), dtype='int32') mincx = min(allChunksArray[:,0]) maxcx = max(allChunksArray[:,0]) mincz = min(allChunksArray[:,1]) From 534565e7171120cb7a94aef4cfa90b4526594c34 Mon Sep 17 00:00:00 2001 From: David Vierra Date: Mon, 29 Nov 2010 18:20:16 -1000 Subject: [PATCH 11/18] don't use int.__mul__ in case the inputs are longs --- box.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/box.py b/box.py index be81ce5..01df695 100644 --- a/box.py +++ b/box.py @@ -91,7 +91,7 @@ class BoundingBox (object): maximum = property(getMaximum, None, None, "The endpoint of the box; origin plus size.") - def getVolume(self): return reduce(int.__mul__, self.size) + def getVolume(self): return reduce(lambda a,b:a*b, self.size) volume = property(getVolume, None, None, "The volume of the box in blocks") @property From 5bee90b002087d20b82ed8819aa63a22ce8b94d2 Mon Sep 17 00:00:00 2001 From: David Vierra Date: Mon, 29 Nov 2010 18:20:36 -1000 Subject: [PATCH 12/18] use world.containsChunk to check if the chunk exists instead of doing a path test --- mclevel.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mclevel.py b/mclevel.py index 5fbecb2..48f614a 100644 --- a/mclevel.py +++ b/mclevel.py @@ -1464,9 +1464,9 @@ class InfdevChunk(MCLevel): if create: self.create(); else: - if not os.path.exists(self.filename): + if not world.containsChunk(*chunkPosition): raise ChunkNotPresent("File not found: {0}", self.filename) - + def compress(self): if not self.dirty: @@ -3197,6 +3197,9 @@ class ZipSchematic (MCInfdevOldLevel): def saveInPlace(self): raise NotImplementedError, "Cannot save zipfiles yet!" + def containsChunk(self, cx, cz): + return (cx,cz) in self.allChunks + def preloadChunkPaths(self): info( u"Scanning for chunks..." ) self._allChunks = set() From 8716c8994b3d6f79e0933d98f5e91523d2d5e214 Mon Sep 17 00:00:00 2001 From: David Vierra Date: Mon, 29 Nov 2010 21:37:56 -1000 Subject: [PATCH 13/18] actually raise the ChunkMalformed instead of an AttributeError when world isn't found this is kinda sketchy --- mclevel.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mclevel.py b/mclevel.py index 48f614a..672ad62 100644 --- a/mclevel.py +++ b/mclevel.py @@ -340,6 +340,7 @@ class MCLevel(object): players = ["Player"] dimNo = 0; parentWorld = None + world = None @classmethod def isLevel(cls, filename): """Tries to find out whether the given filename can be loaded @@ -425,14 +426,14 @@ class MCLevel(object): self.root_tag = nbt.load(buf=fromstring(data, dtype='uint8')); except Exception, e: error( u"Malformed NBT data in file: {0} ({1})".format(self.filename, e) ) - self.world.malformedChunk(*self.chunkPosition); + if self.world: self.world.malformedChunk(*self.chunkPosition); raise ChunkMalformed, self.filename try: self.shapeChunkData() except KeyError: error( u"Incorrect chunk format in file: " + self.filename ) - self.world.malformedChunk(*self.chunkPosition); + if self.world: self.world.malformedChunk(*self.chunkPosition); raise ChunkMalformed, self.filename self.dataIsPacked = True; From bbb37dbca8e3103f84e62f51833622f0823b0ff7 Mon Sep 17 00:00:00 2001 From: David Vierra Date: Mon, 29 Nov 2010 21:38:41 -1000 Subject: [PATCH 14/18] when a second error happens during the fallback attempt, raise both errors --- mclevel.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/mclevel.py b/mclevel.py index 672ad62..682e247 100644 --- a/mclevel.py +++ b/mclevel.py @@ -704,6 +704,9 @@ class MCLevel(object): ''' info( u"Identifying " + filename ) + class LoadingError(RuntimeError): pass + + if not filename: raise IOError, "File not found: "+filename if not os.path.exists(filename): @@ -769,10 +772,13 @@ class MCLevel(object): except Exception, e: info( u"Error during NBT load: {0!r}".format(e) ) info( u"Fallback: Detected compressed flat block array, yzx ordered " ) - lev = MCJavaLevel(filename, data); - lev.compressed = compressed; - return lev; - + try: + lev = MCJavaLevel(filename, data); + lev.compressed = compressed; + return lev; + except Exception, e2: + raise LoadingError, ("Multiple errors encountered", e, e2) + else: if(MCIndevLevel._isTagLevel(root_tag)): info( u"Detected Indev .mclevel" ) From c9fe2afff1f558093f9efd23c8798c69c0745901 Mon Sep 17 00:00:00 2001 From: David Vierra Date: Mon, 29 Nov 2010 21:40:47 -1000 Subject: [PATCH 15/18] added the routine _makeChunk for internal use --- mclevel.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/mclevel.py b/mclevel.py index 682e247..f3302fe 100644 --- a/mclevel.py +++ b/mclevel.py @@ -2283,22 +2283,26 @@ class MCInfdevOldLevel(MCLevel): if chunks is None: chunks = self.allChunks; return [self.getChunk(cx,cz) for (cx,cz) in chunks if self.containsChunk(cx,cz)] + + def _makeChunk(self, cx,cz): + """return the chunk object at the given position, creating it if necessary. + because loading the chunk is done later, accesses to chunk attributes may + raise ChunkMalformed""" + + if not self.containsChunk(cx,cz): + raise ChunkNotPresent, (cx,cz); + + if not (cx,cz) in self._loadedChunks: + self._loadedChunks[cx,cz] = InfdevChunk(self, (cx, cz)); + + return self._loadedChunks[cx,cz] def getChunk(self, cx, cz): """ read the chunk from disk, load it, and return it. decompression and unpacking is done lazily.""" - if self._allChunks is not None: - if not (cx,cz) in self._allChunks: - raise ChunkNotPresent, (cx,cz); - - if not (cx,cz) in self._loadedChunks: - self._loadedChunks[cx,cz] = InfdevChunk(self, (cx, cz)); - - - #raise ChunkNotPresent, "Chunk {0} not present".format((cx,cz)) - c = self._loadedChunks[cx,cz] + c = self._makeChunk(cx,cz) c.load(); if not (cx,cz) in self._loadedChunks: raise ChunkMalformed, "Chunk {0} malformed".format((cx,cz)) From 251b81703b9469959e5ec0950c1831eead61fdbd Mon Sep 17 00:00:00 2001 From: David Vierra Date: Mon, 29 Nov 2010 22:01:00 -1000 Subject: [PATCH 16/18] use _makeChunk to create unloaded chunk objects relight-all failed because it was scanning _loadedChunks when it should have been testing for the chunks' existence also, generator comprehensions --- mclevel.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mclevel.py b/mclevel.py index f3302fe..f686c3b 100644 --- a/mclevel.py +++ b/mclevel.py @@ -2337,11 +2337,12 @@ class MCInfdevOldLevel(MCLevel): startTime = datetime.now(); if dirtyChunks is None: - dirtyChunks = filter(lambda x: x.needsLighting, self._loadedChunks.values()); + dirtyChunks = (ch for ch in self._loadedChunks.itervalues() if ch.needsLighting) else: - dirtyChunks = [self._loadedChunks[c] for c in dirtyChunks if c in self._loadedChunks]; - - dirtyChunks = sorted(list(dirtyChunks), key=lambda x:x.chunkPosition) + dirtyChunks = (self._makeChunk(*c) for c in dirtyChunks if self.containsChunk(*c)) + + dirtyChunks = sorted(dirtyChunks, key=lambda x:x.chunkPosition) + #at 150k per loaded chunk, maxLightingChunks = 4000 From f724a5ef943fa4184f94cbc84b1a43b68d33ace7 Mon Sep 17 00:00:00 2001 From: David Vierra Date: Mon, 29 Nov 2010 22:12:42 -1000 Subject: [PATCH 17/18] cosmetic changes --- mclevel.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mclevel.py b/mclevel.py index f686c3b..d0f61f5 100644 --- a/mclevel.py +++ b/mclevel.py @@ -2480,14 +2480,14 @@ class MCInfdevOldLevel(MCLevel): print "Chunk error during relight, chunk skipped: ", e continue; - for dir,dx,dy,dz in ( (FaceXDecreasing,-1,0,0), - (FaceXIncreasing,1,0,0), - (FaceZDecreasing,0,0, -1), - (FaceZIncreasing,0,0, 1) ): - try: - neighboringChunks[dir] = self.getChunk(cx+dx,cz+dz) - except (ChunkNotPresent, ChunkMalformed): - neighboringChunks[dir] = zeroChunk; + for dir,dx,dz in ( (FaceXDecreasing,-1,0), + (FaceXIncreasing,1,0), + (FaceZDecreasing,0, -1), + (FaceZIncreasing,0, 1) ): + try: + neighboringChunks[dir] = self.getChunk(cx+dx,cz+dz) + except (ChunkNotPresent, ChunkMalformed): + neighboringChunks[dir] = zeroChunk; chunkLa = la[chunk.Blocks]+1; From 6d66f3421c400496aaac4d6639a20e4499dfae18 Mon Sep 17 00:00:00 2001 From: David Vierra Date: Mon, 29 Nov 2010 22:34:52 -1000 Subject: [PATCH 18/18] collect pass/fail messages and print a summary after the run --- run_regression_test.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/run_regression_test.py b/run_regression_test.py index 91c7592..4f0aa1a 100755 --- a/run_regression_test.py +++ b/run_regression_test.py @@ -169,6 +169,9 @@ def main(argv): with untared_content("regression_test/alpha.tar.gz") as directory: test_data = os.path.join(directory, "alpha") + passes = [] + fails = [] + for func, name, sha, args in alpha_tests: if any(fnmatch.fnmatch(name, x) for x in do_these_regressions): if options.profile: @@ -177,9 +180,15 @@ def main(argv): try: func(test_data, sha, args) except RegressionError, e: - print "Regression {0} failed: {1}".format(name, e) + fails.append( "Regression {0} failed: {1}".format(name, e) ) + print fails[-1] else: - print "Regression {0!r} complete.".format(name) + passes.append( "Regression {0!r} complete.".format(name) ) + print passes[-1] + + print "{0} tests passed.".format(len(passes)) + for line in fails: print line; + if __name__ == '__main__': sys.exit(main(sys.argv))