extend pmerge.py to validate and/or correct hashes

direct/src/p3d/FileSpec.py

@@ -1,4 +1,5 @@
 import os
+import time
 from pandac.PandaModules import Filename, HashVal, VirtualFileSystem
 
 class FileSpec:
@@ -88,13 +89,18 @@ class FileSpec:
             xelement.SetAttribute('hash', self.hash)
             
     def quickVerify(self, packageDir = None, pathname = None,
-                    notify = None):
+                    notify = None, correctSelf = False):
         """ Performs a quick test to ensure the file has not been
         modified.  This test is vulnerable to people maliciously
         attempting to fool the program (by setting datestamps etc.).
 
-        Returns true if it is intact, false if it needs to be
+        if correctSelf is True, then any discrepency is corrected by
-        redownloaded. """
+        updating the appropriate fields internally, making the
+        assumption that the file on disk is the authoritative version.
+
+        Returns true if it is intact, false if it is incorrect.  If
+        correctSelf is true, raises OSError if the self-update is
+        impossible (for instance, because the file does not exist)."""
 
         if not pathname:
             pathname = Filename(packageDir, self.filename)
@@ -104,12 +110,16 @@ class FileSpec:
             # If the file is missing, the file fails.
             if notify:
                 notify.debug("file not found: %s" % (pathname))
+                if correctSelf:
+                    raise
             return False
 
         if st.st_size != self.size:
             # If the size is wrong, the file fails.
             if notify:
                 notify.debug("size wrong: %s" % (pathname))
+            if correctSelf:
+                self.__correctHash(packageDir, pathname, st, notify)
             return False
 
         if st.st_mtime == self.timestamp:
@@ -129,6 +139,8 @@ class FileSpec:
             if notify:
                 notify.debug("hash check wrong: %s" % (pathname))
                 notify.debug("  found %s, expected %s" % (self.actualFile.hash, self.hash))
+            if correctSelf:
+                self.__correctHash(packageDir, pathname, st, notify)
             return False
 
         if notify:
@@ -137,6 +149,11 @@ class FileSpec:
         # The hash is OK after all.  Change the file's timestamp back
         # to what we expect it to be, so we can quick-verify it
         # successfully next time.
+        if correctSelf:
+            # Or update our own timestamp.
+            self.__correctTimestamp(pathname, st, notify)
+            return False
+        else:
             self.__updateTimestamp(pathname, st)
 
         return True
@@ -194,6 +211,14 @@ class FileSpec:
         except OSError:
             pass
 
+    def __correctTimestamp(self, pathname, st, notify):
+        """ Corrects the internal timestamp to match the one on
+        disk. """
+        if notify:
+            notify.info("Correcting timestamp of %s to %d (%s)" % (
+                self.filename, st.st_mtime, time.asctime(time.localtime(st.st_mtime))))
+        self.timestamp = st.st_mtime
+
     def checkHash(self, packageDir, pathname, st):
         """ Returns true if the file has the expected md5 hash, false
         otherwise.  As a side effect, stores a FileSpec corresponding
@@ -206,3 +231,14 @@ class FileSpec:
 
         return (fileSpec.hash == self.hash)
 
+    def __correctHash(self, packageDir, pathname, st, notify):
+        """ Corrects the internal hash to match the one on disk. """
+        if not self.actualFile:
+            self.checkHash(packageDir, pathname, st)
+            
+        if notify:
+            notify.info("Correcting hash %s to %s" % (
+                self.filename, self.actualFile.hash))
+        self.hash = self.actualFile.hash
+        self.size = self.actualFile.size
+        self.timestamp = self.actualFile.timestamp

direct/src/p3d/PackageMerger.py

@@ -1,5 +1,6 @@
 from direct.p3d.FileSpec import FileSpec
 from direct.p3d.SeqValue import SeqValue
+from direct.directnotify.DirectNotifyGlobal import *
 from pandac.PandaModules import *
 import copy
 import shutil
@@ -16,6 +17,8 @@ class PackageMerger:
     most recent timestamp (indicated in the contents.xml file) is
     always the most current version of the file. """
 
+    notify = directNotify.newCategory("PackageMerger")
+ 
     class PackageEntry:
         """ This corresponds to a <package> entry in the contents.xml
         file. """
@@ -42,6 +45,10 @@ class PackageMerger:
             self.descFile = FileSpec()
             self.descFile.loadXml(xpackage)
 
+            self.validatePackageContents()
+            
+            self.descFile.quickVerify(packageDir = self.sourceDir, notify = PackageMerger.notify, correctSelf = True)
+
             self.packageSeq = SeqValue()
             self.packageSeq.loadXml(xpackage, 'seq')
             self.packageSetVer = SeqValue()
@@ -52,6 +59,7 @@ class PackageMerger:
             if ximport:
                 self.importDescFile = FileSpec()
                 self.importDescFile.loadXml(ximport)
+                self.importDescFile.quickVerify(packageDir = self.sourceDir, notify = PackageMerger.notify, correctSelf = True)
 
         def makeXml(self):
             """ Returns a new TiXmlElement. """
@@ -75,6 +83,50 @@ class PackageMerger:
             
             return xpackage
 
+        def validatePackageContents(self):
+            """ Validates the contents of the package directory itself
+            against the expected hashes and timestamps.  Updates
+            hashes and timestamps where needed. """
+
+            if self.solo:
+                return
+
+            needsChange = False
+            packageDescFullpath = Filename(self.sourceDir, self.descFile.filename)
+            packageDir = Filename(packageDescFullpath.getDirname())
+            doc = TiXmlDocument(packageDescFullpath.toOsSpecific())
+            if not doc.LoadFile():
+                message = "Could not read XML file: %s" % (self.descFile.filename)
+                raise OSError, message
+
+            xpackage = doc.FirstChildElement('package')
+            if not xpackage:
+                message = "No package definition: %s" % (self.descFile.filename)
+                raise OSError, message
+
+            xcompressed = xpackage.FirstChildElement('compressed_archive')
+            if xcompressed:
+                spec = FileSpec()
+                spec.loadXml(xcompressed)
+                if not spec.quickVerify(packageDir = packageDir, notify = PackageMerger.notify, correctSelf = True):
+                    spec.storeXml(xcompressed)
+                    needsChange = True
+
+            xpatch = xpackage.FirstChildElement('patch')
+            while xpatch:
+                spec = FileSpec()
+                spec.loadXml(xpatch)
+                if not spec.quickVerify(packageDir = packageDir, notify = PackageMerger.notify, correctSelf = True):
+                    spec.storeXml(xpatch)
+                    needsChange = True
+
+                xpatch = xpatch.NextSiblingElement('patch')
+
+            if needsChange:
+                PackageMerger.notify.info("Rewriting %s" % (self.descFile.filename))
+                doc.SaveFile()
+                self.descFile.quickVerify(packageDir = self.sourceDir, notify = PackageMerger.notify, correctSelf = True)
+
     # PackageMerger constructor
     def __init__(self, installDir):
         self.installDir = installDir
@@ -161,7 +213,7 @@ class PackageMerger:
         there. """
 
         dirname = Filename(pe.descFile.filename).getDirname()
-        print "copying %s" % (dirname)
+        self.notify.info("copying %s" % (dirname))
         sourceDirname = Filename(pe.sourceDir, dirname)
         targetDirname = Filename(self.installDir, dirname)
 
@@ -208,6 +260,13 @@ class PackageMerger:
             # Copying a regular file.
             sourceFilename.copyTo(targetFilename)
 
+            # Also try to copy the timestamp, but don't fuss too much
+            # if it doesn't work.
+            try:
+                st = os.stat(sourceFilename.toOsSpecific())
+                os.utime(targetFilename.toOsSpecific(), (st.st_atime, st.st_mtime))
+            except OSError:
+                pass
 
     def merge(self, sourceDir):
         """ Adds the contents of the indicated source directory into

direct/src/p3d/pmerge.py

@@ -4,7 +4,9 @@ usageText = """
 
 This script can be used to merge together the contents of two or more
 separately-built stage directories, built independently via ppackage,
-or via Packager.py.
+or via Packager.py.  This script also verifies the hash, file size,
+and timestamp values in the stage directory as it runs, so it can be
+run on a single standalone directory just to perform this validation.
 
 This script is actually a wrapper around Panda's PackageMerger.py.
 
@@ -24,7 +26,9 @@ Options:
   -i install_dir
      The full path to the final install directory.  This may also
      contain some pre-existing contents; if so, it is merged with all
-     of the input directories as well.
+     of the input directories as well.  The contents of this directory
+     are checked for self-consistency with regards to hashes and
+     timestamps.
 
   -h
      Display this help
@@ -62,9 +66,11 @@ inputDirs = []
 for arg in args:
     inputDirs.append(Filename.fromOsSpecific(arg))
 
-if not inputDirs:
+# It's now legal to have no input files if you only want to verify
-    print "no input directories specified."
+# timestamps and hashes.
-    sys.exit(1)
+## if not inputDirs:
+##     print "no input directories specified."
+##     sys.exit(1)
 
 try:
     pm = PackageMerger.PackageMerger(installDir)