wip: packager

direct/src/showutil/FreezeTool.py

@@ -359,6 +359,12 @@ class Freezer:
     MTExclude = 2
     MTForbid = 3
 
+    class ModuleDef:
+        def __init__(self, token, moduleName, filename = None):
+            self.token = token
+            self.moduleName = moduleName
+            self.filename = filename
+
     def __init__(self, previous = None, debugLevel = 0):
         # Normally, we are freezing for our own platform.  Change this
         # if untrue.
@@ -378,10 +384,6 @@ class Freezer:
         self.objectExtension = '.o'
         if self.platform == 'win32':
             self.objectExtension = '.obj'
-        
-        # True to compile to an executable, false to compile to a dll.  If
-        # setMain() is called, this is automatically set to True.
-        self.compileToExe = False
 
         # Change any of these to change the generated startup and glue
         # code.
@@ -390,32 +392,51 @@ class Freezer:
         self.mainInitCode = mainInitCode
         self.frozenExtensions = frozenExtensions
 
+        # Set this true to encode Python files in a Multifile as their
+        # original source if possible, or false to encode them as
+        # compiled pyc or pyo files.  This has no effect on frozen exe
+        # or dll's; those are always stored with compiled code.
+        self.storePythonSource = False
 
         # End of public interface.  These remaining members should not
         # be directly manipulated by callers.
         self.previousModules = {}
         self.modules = {}
 
+        self.virtualModules = {}
+
         if previous:
             self.previousModules = dict(previous.modules)
             self.modules = dict(previous.modules)
 
-        self.mainModule = None
         self.mf = None
 
         # Make sure we know how to find "direct".
         if direct.__path__:
             modulefinder.AddPackagePath('direct', direct.__path__[0])
 
+    def excludeFrom(self, freezer):
+        """ Excludes all modules that have already been processed by
+        the indicated FreezeTool.  This is equivalent to passing the
+        indicated FreezeTool object as previous to this object's
+        constructor, but it may be called at any point during
+        processing. """
+
+        for key, value in freezer.modules:
+            self.previousModules[key] = value
+            self.modules[key] = value
+
     def excludeModule(self, moduleName, forbid = False):
         """ Adds a module to the list of modules not to be exported by
         this tool.  If forbid is true, the module is furthermore
         forbidden to be imported, even if it exists on disk. """
-        
+
+        assert self.mf == None
+
         if forbid:
-            self.modules[moduleName] = self.MTForbid
+            self.modules[moduleName] = self.ModuleDef(self.MTForbid, moduleName)
         else:
-            self.modules[moduleName] = self.MTExclude
+            self.modules[moduleName] = self.ModuleDef(self.MTExclude, moduleName)
 
     def handleCustomPath(self, moduleName):
         """ Indicates a module that may perform runtime manipulation
@@ -467,77 +488,95 @@ class Freezer:
         else:
             return None
             
-    def addModule(self, moduleName, implicit = False):
+    def addModule(self, moduleName, implicit = False, newName = None,
+                  filename = None):
         """ Adds a module to the list of modules to be exported by
         this tool.  If implicit is true, it is OK if the module does
         not actually exist.
 
+        newName is the name to call the module when it appears in the
+        output.  The default is the same name it had in the original.
+        Use caution when renaming a module; if another module imports
+        this module by its original name, the module will need to be
+        duplicated in the output, a copy for each name.
+
         The module name may end in ".*", which means to add all of the
         .py files (other than __init__.py) in a particular directory.
         It may also end in ".*.*", which means to cycle through all
         directories within a particular directory.
         """
 
+        assert self.mf == None
+
+        if not newName:
+            newName = moduleName
         moduleName = moduleName.replace("/", ".").replace("direct.src", "direct")
+        newName = newName.replace("/", ".").replace("direct.src", "direct")
         if implicit:
             token = self.MTAuto
         else:
             token = self.MTInclude
 
         if moduleName.endswith('.*'):
+            assert(newName.endswith('.*'))
             # Find the parent module, so we can get its directory.
             parentName = moduleName[:-2]
-            parentNames = [parentName]
+            newParentName = newName[:-2]
+            parentNames = [(parentName, newParentName)]
 
             if parentName.endswith('.*'):
+                assert(newParentName.endswith('.*'))
                 # Another special case.  The parent name "*" means to
                 # return all possible directories within a particular
                 # directory.
 
                 topName = parentName[:-2]
+                newTopName = newParentName[:-2]
                 parentNames = []
                 for dirname in self.getModulePath(topName):
-                    for filename in os.listdir(dirname):
-                        if os.path.exists(os.path.join(dirname, filename, '__init__.py')):
-                            parentName = '%s.%s' % (topName, filename)
+                    for basename in os.listdir(dirname):
+                        if os.path.exists(os.path.join(dirname, basename, '__init__.py')):
+                            parentName = '%s.%s' % (topName, basename)
+                            newParentName = '%s.%s' % (newTopName, basename)
                             if self.getModulePath(parentName):
-                                parentNames.append(parentName)
+                                parentNames.append((parentName, newParentName))
 
-            for parentName in parentNames:
+            for parentName, newParentName in parentNames:
                 path = self.getModulePath(parentName)
 
                 if path == None:
                     # It's actually a regular module.
-                    self.modules[parentName] = token
+                    self.modules[newParentName] = self.ModuleDef(token, parentName)
 
                 else:
                     # Now get all the py files in the parent directory.
                     for dirname in path:
-                        for filename in os.listdir(dirname):
-                            if '-' in filename:
+                        for basename in os.listdir(dirname):
+                            if '-' in basename:
                                 continue
-                            if filename.endswith('.py') and filename != '__init__.py':
-                                moduleName = '%s.%s' % (parentName, filename[:-3])
-                                self.modules[moduleName] = token
+                            if basename.endswith('.py') and basename != '__init__.py':
+                                moduleName = '%s.%s' % (parentName, basename[:-3])
+                                newName = '%s.%s' % (newParentName, basename[:-3])
+                                self.modules[newName] = self.ModuleDef(token, moduleName)
         else:
             # A normal, explicit module name.
-            self.modules[moduleName] = token
-
-    def setMain(self, moduleName):
-        moduleName = moduleName.replace("/", ".").replace("direct.src", "direct")
-        self.addModule(moduleName)
-        self.mainModule = moduleName
-        self.compileToExe = True
+            self.modules[newName] = self.ModuleDef(token, moduleName, filename = filename)
 
     def done(self):
+        """ Call this method after you have added all modules with
+        addModule().  You may then call generateCode() or
+        writeMultifile() to dump the resulting output.  After a call
+        to done(), you may not add any more modules until you call
+        reset(). """
+        
         assert self.mf == None
 
-        if self.compileToExe:
-            # Ensure that each of our required startup modules is
-            # on the list.
+        # If we have a __main__ module, we also need to implicitly
+        # bring in Python's startup modules.
+        if '__main__' in self.modules:
             for moduleName in startupModules:
                 if moduleName not in self.modules:
-                    self.modules[moduleName] = self.MTAuto
+                    self.modules[moduleName] = self.ModuleDef(self.MTAuto, moduleName)
 
         # Excluding a parent module also excludes all its children.
         # Walk through the list in sorted order, so we reach children
@@ -549,91 +588,131 @@ class Freezer:
         excludeDict = {}
         includes = []
         autoIncludes = []
-        for moduleName, token in names:
-            if '.' in moduleName:
-                parentName, baseName = moduleName.rsplit('.', 1)
+        origToNewName = {}
+        for newName, mdef in names:
+            token = mdef.token
+            origToNewName[mdef.moduleName] = newName
+            if '.' in newName:
+                parentName, baseName = newName.rsplit('.', 1)
                 if parentName in excludeDict:
                     token = excludeDict[parentName]
             
             if token == self.MTInclude:
-                includes.append(moduleName)
+                includes.append(mdef)
             elif token == self.MTAuto:
-                autoIncludes.append(moduleName)
+                autoIncludes.append(mdef)
             elif token == self.MTExclude or token == self.MTForbid:
-                excludes.append(moduleName)
-                excludeDict[moduleName] = token
+                excludes.append(mdef.moduleName)
+                excludeDict[mdef.moduleName] = token
 
         self.mf = modulefinder.ModuleFinder(excludes = excludes)
 
         # Attempt to import the explicit modules into the modulefinder.
-        for moduleName in includes:
-            self.mf.import_hook(moduleName)
+        for mdef in includes:
+            self.__loadModule(mdef)
 
         # Also attempt to import any implicit modules.  If any of
-        # these fail to import, we don't care.
-        for moduleName in autoIncludes:
+        # these fail to import, we don't really care.
+        for mdef in autoIncludes:
             try:
-                self.mf.import_hook(moduleName)
+                self.__loadModule(mdef)
             except ImportError:
                 pass
 
         # Now, any new modules we found get added to the export list.
-        for moduleName in self.mf.modules.keys():
-            if moduleName not in self.modules:
-                self.modules[moduleName] = self.MTAuto
+        for origName in self.mf.modules.keys():
+            if origName not in origToNewName:
+                self.modules[origName] = self.ModuleDef(self.MTAuto, origName)
+                            
+            elif origName not in self.modules:
+                print "Module %s renamed to %s, but imported directly with its original name" % (
+                    origName, origToNewName[origName])
+                self.modules[origName] = self.ModuleDef(self.MTAuto, origName)
 
         missing = []
-        for moduleName in self.mf.any_missing():
-            if moduleName in startupModules:
+        for origName in self.mf.any_missing():
+            if origName in startupModules:
                 continue
-            if moduleName in self.previousModules:
+            if origName in self.previousModules:
                 continue
 
             # This module is missing.  Let it be missing in the
             # runtime also.
-            self.modules[moduleName] = self.MTExclude
+            self.modules[origName] = self.ModuleDef(self.MTExclude, origName)
 
-            if moduleName in okMissing:
+            if origName in okMissing:
                 # If it's listed in okMissing, don't even report it.
                 continue
 
-            prefix = moduleName.split('.')[0]
+            prefix = origName.split('.')[0]
             if prefix not in sourceTrees:
                 # If it's in not one of our standard source trees, assume
                 # it's some wacky system file we don't need.
                 continue
                 
-            missing.append(moduleName)
+            missing.append(origName)
                 
         if missing:
             error = "There are some missing modules: %r" % missing
             print error
             raise StandardError, error
 
-    def mangleName(self, moduleName):
-        return 'M_' + moduleName.replace('.', '__')
+    def __loadModule(self, mdef):
+        """ Adds the indicated module to the modulefinder. """
+        
+        if mdef.filename:
+            # If it has a filename, then we found it as a file on
+            # disk.  In this case, the moduleName may not be accurate
+            # and useful, so load it as a file instead.
 
-    def __getModuleNames(self):
+            pathname = mdef.filename.toOsSpecific()
+            fp = open(pathname, modulefinder.READ_MODE)
+            stuff = ("", "r", imp.PY_SOURCE)
+            self.mf.load_module(mdef.moduleName, fp, pathname, stuff)
+
+        else:
+            # Otherwise, we can just import it normally.
+            self.mf.import_hook(mdef.moduleName)
+
+    def reset(self):
+        """ After a previous call to done(), this resets the
+        FreezeTool object for a new pass.  More modules may be added
+        and dumped to a new target.  Previously-added modules are
+        remembered and will not be dumped again. """
+        
+        self.mf = None
+        self.previousModules = dict(self.modules)
+
+    def mangleName(self, moduleName):
+        return 'M_' + moduleName.replace('.', '__').replace('-', '_')
+
+    def __getModuleDefs(self):
         # Collect a list of all of the modules we will be explicitly
         # referencing.
-        moduleNames = []
+        moduleDefs = []
 
-        for moduleName, token in self.modules.items():
-            prevToken = self.previousModules.get(moduleName, None)
+        for newName, mdef in self.modules.items():
+            token = mdef.token
+            prev = self.previousModules.get(newName, None)
             if token == self.MTInclude or token == self.MTAuto:
                 # Include this module (even if a previous pass
                 # excluded it).  But don't bother if we exported it
                 # previously.
-                if prevToken != self.MTInclude and prevToken != self.MTAuto:
-                    if moduleName in self.mf.modules or \
-                       moduleName in startupModules:
-                        moduleNames.append(moduleName)
+                if prev and \
+                   (prev.token == self.MTInclude or prev.token == self.MTAuto):
+                    # Previously exported.
+                    pass
+                else:
+                    if newName in self.mf.modules or \
+                       newName in startupModules or \
+                       mdef.filename:
+                        moduleDefs.append((newName, mdef))
             elif token == self.MTForbid:
-                if prevToken != self.MTForbid:
-                    moduleNames.append(moduleName)
+                if not prev or prev.token != self.MTForbid:
+                    moduleDefs.append((newName, mdef))
 
-        moduleNames.sort()
-        return moduleNames
+        moduleDefs.sort()
+        return moduleDefs
 
     def __replacePaths(self):
         # Build up the replacement pathname table, so we can eliminate
@@ -672,45 +751,95 @@ class Freezer:
         if str not in moduleDirs:
             # Add an implicit __init__.py file.
             moduleName = '.'.join(dirnames)
-            filename = '/'.join(dirnames) + '/__init__.py'
+            filename = '/'.join(dirnames) + '/__init__'
 
-            stream = StringStream('')
-            multifile.addSubfile(filename, stream, 0)
-            multifile.flush()
+            if self.storePythonSource:
+                filename += '.py'
+                stream = StringStream('')
+                multifile.addSubfile(filename, stream, 0)
+                multifile.flush()
+            else:
+                if __debug__:
+                    filename += '.pyc'
+                else:
+                    filename += '.pyo'
+                code = compile('', moduleName, 'exec')
+                self.__addPyc(multifile, filename, code)
 
             moduleDirs[str] = True
             self.__addPythonDirs(multifile, moduleDirs, dirnames[:-1])
 
-    def __addPythonFile(self, multifile, moduleDirs, moduleName):
+    def __addPythonFile(self, multifile, moduleDirs, moduleName, mdef):
         """ Adds the named module to the multifile as a .pyc file. """
-        module = self.mf.modules.get(moduleName, None)
-        if getattr(module, '__path__', None) is not None:
-            # It's actually a package.  In this case, we really write
-            # the file moduleName/__init__.py.
-            moduleName += '.__init__'
 
         # First, split the module into its subdirectory names.
         dirnames = moduleName.split('.')
         self.__addPythonDirs(multifile, moduleDirs, dirnames[:-1])
 
+        filename = '/'.join(dirnames)
+
+        module = self.mf.modules.get(moduleName, None)
+        if getattr(module, '__path__', None) is not None:
+            # It's actually a package.  In this case, we really write
+            # the file moduleName/__init__.py.
+            filename += '/__init__'
+
+            # Ensure we don't have an implicit filename from above.
+            multifile.removeSubfile(filename + '.py')
+            if __debug__:
+                multifile.removeSubfile(filename + '.pyc')
+            else:
+                multifile.removeSubfile(filename + '.pyo')
+
         # Attempt to add the original source file if we can.
-        if getattr(module, '__file__', None):
-            sourceFilename = Filename.fromOsSpecific(module.__file__)
-            sourceFilename.setExtension("py")
-            if sourceFilename.exists():
-                filename = '/'.join(dirnames) + '.py'
+        if self.storePythonSource:
+            sourceFilename = None
+            if mdef.filename and mdef.filename.getExtension() == "py":
+                sourceFilename = mdef.filename
+            elif getattr(module, '__file__', None):
+                sourceFilename = Filename.fromOsSpecific(module.__file__)
+                sourceFilename.setExtension("py")
+            
+            if sourceFilename and sourceFilename.exists():
+                filename += '.py'
                 multifile.addSubfile(filename, sourceFilename, 0)
                 return
 
         # If we can't find the source file, add the compiled pyc instead.
-        filename = '/'.join(dirnames) + '.pyc'
-        code = getattr(module, "__code__", None)
+        if __debug__:
+            filename += '.pyc'
+        else:
+            filename += '.pyo'
+
+        if module:
+            # Get the compiled code directly from the module object.
+            code = getattr(module, "__code__", None)
+        else:
+            # Read the code from the source file and compile it on-the-fly.
+            sourceFilename = None
+            if mdef.filename and mdef.filename.getExtension() == "py":
+                sourceFilename = mdef.filename
+                source = open(sourceFilename.toOsSpecific(), 'r').read()
+                if source and source[-1] != '\n':
+                    source = source + '\n'
+                code = compile(source, sourceFilename.cStr(), 'exec')
+
         self.__addPyc(multifile, filename, code)
 
+    def addToMultifile(self, multifile):
+        """ After a call to done(), this stores all of the accumulated
+        python code into the indicated Multifile. """
+
+        moduleDirs = {}
+        for moduleName, mdef in self.__getModuleDefs():
+            if mdef.token != self.MTForbid:
+                self.__addPythonFile(multifile, moduleDirs, moduleName, mdef)
     
     def writeMultifile(self, mfname):
-        """Instead of generating a frozen file, put all of the Python
-        code in a multifile. """
+        """ After a call to done(), this stores all of the accumulated
+        python code into a Multifile with the indicated filename,
+        including the extension. """
+
         self.__replacePaths()
 
         Filename(mfname).unlink()
@@ -718,37 +847,49 @@ class Freezer:
         if not multifile.openReadWrite(mfname):
             raise StandardError
 
-        moduleDirs = {}
-        for moduleName in self.__getModuleNames():
-            token = self.modules[moduleName]
-            if token != self.MTForbid:
-                self.__addPythonFile(multifile, moduleDirs, moduleName)
+        self.addToMultifile(multifile)
 
         multifile.flush()
         multifile.repack()
 
-    def generateCode(self, basename):
+    def generateCode(self, basename, compileToExe = False):
+        """ After a call to done(), this freezes all of the
+        accumulated python code into either an executable program (if
+        compileToExe is true) or a dynamic library (if compileToExe is
+        false).  The basename is the name of the file to write,
+        without the extension.
+
+        The return value is the newly-generated filename, including
+        the extension. """
+        
+        if compileToExe:
+            # We must have a __main__ module to make an exe file.
+            if not self.__writingModule('__main__'):
+                message = "Can't generate an executable without a __main__ module."
+                raise StandardError, message
+
         self.__replacePaths()
 
         # Now generate the actual export table.
         moduleDefs = []
         moduleList = []
         
-        for moduleName in self.__getModuleNames():
-            token = self.modules[moduleName]
+        for moduleName, mdef in self.__getModuleDefs():
+            token = mdef.token
+            origName = mdef.moduleName
             if token == self.MTForbid:
                 # Explicitly disallow importing this module.
                 moduleList.append(self.makeForbiddenModuleListEntry(moduleName))
             else:
                 assert token != self.MTExclude
                 # Allow importing this module.
-                module = self.mf.modules.get(moduleName, None)
+                module = self.mf.modules.get(origName, None)
                 code = getattr(module, "__code__", None)
                 if not code and moduleName in startupModules:
                     # Forbid the loading of this startup module.
                     moduleList.append(self.makeForbiddenModuleListEntry(moduleName))
                 else:
-                    if moduleName in sourceTrees:
+                    if origName in sourceTrees:
                         # This is one of our Python source trees.
                         # These are a special case: we don't compile
                         # the __init__.py files within them, since
@@ -763,13 +904,10 @@ class Freezer:
                         mangledName = self.mangleName(moduleName)
                         moduleDefs.append(self.makeModuleDef(mangledName, code))
                         moduleList.append(self.makeModuleListEntry(mangledName, code, moduleName, module))
-                        if moduleName == self.mainModule:
-                            # Add a special entry for __main__.
-                            moduleList.append(self.makeModuleListEntry(mangledName, code, '__main__', module))
 
         filename = basename + self.sourceExtension
 
-        if self.compileToExe:
+        if compileToExe:
             code = self.frozenMainCode
             if self.platform == 'win32':
                 code += self.frozenDllMainCode
@@ -897,3 +1035,18 @@ class Freezer:
     def makeForbiddenModuleListEntry(self, moduleName):
         return '  {"%s", NULL, 0},' % (moduleName)
 
+
+    def __writingModule(self, moduleName):
+        """ Returns true if we are outputting the named module in this
+        pass, false if we have already output in a previous pass, or
+        if it is not yet on the output table. """
+        
+        mdef = self.modules.get(moduleName, (None, None))
+        token = mdef.token
+        if token != self.MTAuto and token != self.MTInclude:
+            return False
+
+        if moduleName in self.previousModules:
+            return False
+
+        return True

direct/src/showutil/Packager.py

@@ -0,0 +1,469 @@
+""" This module is used to build a "Package", a collection of files
+within a Panda3D Multifile, which can be easily be downloaded and/or
+patched onto a client machine, for the purpose of running a large
+application. """
+
+import sys
+import os
+import glob
+import marshal
+import new
+from direct.showutil import FreezeTool
+from direct.directnotify.DirectNotifyGlobal import *
+from pandac.PandaModules import *
+
+class PackagerError(StandardError):
+    pass
+
+class OutsideOfPackageError(PackagerError):
+    pass
+
+class ArgumentError(PackagerError):
+    pass
+
+class Packager:
+    notify = directNotify.newCategory("Packager")
+
+    class PackFile:
+        def __init__(self, filename, newName = None, deleteTemp = False):
+            self.filename = filename
+            self.newName = newName
+            self.deleteTemp = deleteTemp
+
+    class Package:
+        def __init__(self, packageName):
+            self.packageName = packageName
+            self.version = 'dev'
+            self.files = []
+
+            # This records the current list of modules we have added so
+            # far.
+            self.freezer = FreezeTool.Freezer()
+
+        def close(self):
+            """ Writes out the contents of the current package. """
+
+            packageFilename = self.packageName
+            packageFilename += '_' + self.version
+            packageFilename += '.mf'
+            
+            try:
+                os.unlink(packageFilename)
+            except OSError:
+                pass
+            
+            multifile = Multifile()
+            multifile.openReadWrite(packageFilename)
+
+            sourceFilename = {}
+            targetFilename = {}
+            for file in self.files:
+                if not file.newName:
+                    file.newName = file.filename
+                sourceFilename[file.filename] = file
+                targetFilename[file.newName] = file
+
+            for file in self.files:
+                ext = file.filename.getExtension()
+                if ext == 'py':
+                    self.addPyFile(file)
+                else:
+                    # An ordinary file.
+                    multifile.addSubfile(file.newName, file.filename, 0)
+
+            # Pick up any unfrozen Python files.
+            self.freezer.done()
+            self.freezer.addToMultifile(multifile)
+
+            multifile.repack()
+            multifile.close()
+
+            # Now that all the files have been packed, we can delete
+            # the temporary files.
+            for file in self.files:
+                if file.deleteTemp:
+                    os.unlink(file.filename)
+
+        def addPyFile(self, file):
+            """ Adds the indicated python file, identified by filename
+            instead of by module name, to the package. """
+
+            # Convert the raw filename back to a module name, so we
+            # can see if we've already loaded this file.  We assume
+            # that all Python files within the package will be rooted
+            # at the top of the package.
+
+            filename = file.newName.rsplit('.', 1)[0]
+            moduleName = filename.replace("/", ".")
+            if moduleName.endswith('.__init__'):
+                moduleName = moduleName.rsplit('.', 1)[0]
+
+            if moduleName in self.freezer.modules:
+                # This Python file is already known.  We don't have to
+                # deal with it again.
+                return
+
+            self.freezer.addModule(moduleName, newName = moduleName,
+                                   filename = file.filename)
+
+    def __init__(self):
+
+        # The following are config settings that the caller may adjust
+        # before calling any of the command methods.
+
+        # These should each be a Filename, or None if they are not
+        # filled in.
+        self.installDir = None
+        self.persistDir = None
+
+        # The platform string.
+        self.platform = PandaSystem.getPlatform()
+
+        # Optional signing and encrypting features.
+        self.encryptionKey = None
+        self.prcEncryptionKey = None
+        self.prcSignCommand = None
+
+        # This is a list of filename extensions and/or basenames that
+        # indicate files that should be encrypted within the
+        # multifile.  This provides obfuscation only, not real
+        # security, since the decryption key must be part of the
+        # client and is therefore readily available to any hacker.
+        # Not only is this feature useless, but using it also
+        # increases the size of your patchfiles, since encrypted files
+        # don't patch as tightly as unencrypted files.  But it's here
+        # if you really want it.
+        self.encryptExtensions = ['ptf', 'dna', 'txt', 'dc']
+        self.encryptFiles = []
+
+        # This is the list of DC import suffixes that should be
+        # available to the client.  Other suffixes, like AI and UD,
+        # are server-side only and should be ignored by the Scrubber.
+        self.dcClientSuffixes = ['OV']
+
+    def setup(self):
+        """ Call this method to initialize the class after filling in
+        some of the values in the constructor. """
+
+        # We need a stack of packages for managing begin_package
+        # .. end_package.
+        self.packageStack = []
+        self.currentPackage = None
+
+        # The persist dir is the directory in which the results from
+        # past publishes are stored so we can generate patches against
+        # them.  There must be a nonempty directory name here.
+        assert(self.persistDir)
+
+        # If the persist dir names an empty or nonexistent directory,
+        # we will be generating a brand new publish with no previous
+        # patches.
+        self.persistDir.makeDir()
+
+        # Within the persist dir, we make a temporary holding dir for
+        # generating multifiles.
+        self.mfTempDir = Filename(self.persistDir, Filename('mftemp/'))
+        #self.mfTempDir.makeDir()
+
+        # We also need a temporary holding dir for squeezing py files.
+        self.pyzTempDir = Filename(self.persistDir, Filename('pyz/'))
+        #self.pyzTempDir.makeDir()
+
+        # Change to the persist directory so the temp files will be
+        # created there
+        os.chdir(self.persistDir.toOsSpecific())
+
+
+    def readPackageDef(self, packageDef):
+        """ Reads the lines in packageDef and dispatches to the
+        appropriate handler method for each line. """
+
+        self.notify.info('Reading %s' % (packageDef))
+        file = open(packageDef.toOsSpecific())
+        lines = file.readlines()
+        file.close()
+
+        lineNum = [0]
+        def getNextLine(lineNum = lineNum):
+            """
+            Read in the next line of the packageDef
+            """
+            while lineNum[0] < len(lines):
+                line = lines[lineNum[0]].strip()
+                lineNum[0] += 1
+                if not line:
+                    # Skip the line, it was just a blank line
+                    pass
+                elif line[0] == '#':
+                    # Eat python-style comments.
+                    pass
+                else:
+                    # Remove any trailing comment.
+                    hash = line.find(' #')
+                    if hash != -1:
+                        line = line[:hash].strip()
+                    # Return the line as an array split at whitespace.
+                    return line.split()
+
+            # EOF.
+            return None
+
+        # Now start parsing the packageDef lines
+        try:
+            lineList = getNextLine()
+            while lineList:
+                command = lineList[0]
+                try:
+                    methodName = 'parse_%s' % (command)
+                    method = getattr(self, methodName, None)
+                    if method:
+                        method(lineList)
+
+                    else:
+                        message = 'Unknown command %s' % (command)
+                        raise PackagerError, message
+                except ArgumentError:
+                    message = 'Wrong number of arguments for command %s' %(command)
+                    raise ArgumentError, message
+                except OutsideOfPackageError:
+                    message = '%s command encounted outside of package specification' %(command)
+                    raise OutsideOfPackageError, message
+
+                lineList = getNextLine()
+
+        except PackagerError:
+            # Append the line number and file name to the exception
+            # error message.
+            inst = sys.exc_info()[1]
+            inst.args = (inst.args[0] + ' on line %s of %s' % (lineNum[0], packageDef),)
+            raise
+
+    def parse_setenv(self, lineList):
+        """
+        setenv variable value
+        """
+        
+        try:
+            command, variable, value = lineList
+        except ValueError:
+            raise ArgumentNumber
+
+        value = ExecutionEnvironment.expandString(value)
+        ExecutionEnvironment.setEnvironmentVariable(variable, value)
+            
+    def parse_begin_package(self, lineList):
+        """
+        begin_package packageName
+        """
+        
+        try:
+            command, packageName = lineList
+        except ValueError:
+            raise ArgumentNumber
+
+        self.beginPackage(packageName)
+
+    def parse_end_package(self, lineList):
+        """
+        end_package packageName
+        """
+
+        try:
+            command, packageName = lineList
+        except ValueError:
+            raise ArgumentError
+
+        self.endPackage(packageName)
+
+    def parse_module(self, lineList):
+        """
+        module moduleName [newName]
+        """
+        newName = None
+
+        try:
+            if len(lineList) == 2:
+                command, moduleName = lineList
+            else:
+                command, moduleName, newName = lineList
+        except ValueError:
+            raise ArgumentError
+
+        self.module(moduleName, newName = newName)
+
+    def parse_freeze_exe(self, lineList):
+        """
+        freeze_exe path/to/basename
+        """
+
+        try:
+            command, filename = lineList
+        except ValueError:
+            raise ArgumentError
+
+        self.freeze(filename, compileToExe = True)
+
+    def parse_freeze_dll(self, lineList):
+        """
+        freeze_dll path/to/basename
+        """
+
+        try:
+            command, filename = lineList
+        except ValueError:
+            raise ArgumentError
+
+        self.freeze(filename, compileToExe = False)
+
+    def parse_file(self, lineList):
+        """
+        file filename [newNameOrDir]
+        """
+
+        newNameOrDir = None
+
+        try:
+            if len(lineList) == 2:
+                command, filename = lineList
+            else:
+                command, filename, newNameOrDir = lineList
+        except ValueError:
+            raise ArgumentError
+
+        self.file(filename, newNameOrDir = newNameOrDir)
+    
+    def beginPackage(self, packageName):
+        """ Begins a new package specification.  packageName is the
+        basename of the package.  Follow this with a number of calls
+        to file() etc., and close the package with endPackage(). """
+        
+        package = self.Package(packageName)
+        if self.currentPackage:
+            package.freezer.excludeFrom(self.currentPackage.freezer)
+            
+        self.packageStack.append(package)
+        self.currentPackage = package
+
+    def endPackage(self, packageName):
+        """ Closes a package specification.  This actually generates
+        the package file.  The packageName must match the previous
+        call to beginPackage(). """
+        
+        if not self.currentPackage:
+            raise PackagerError, 'unmatched end_package %s' % (packageName)
+        if self.currentPackage.packageName != packageName:
+            raise PackagerError, 'end_package %s where %s expected' % (
+                packageName, self.currentPackage.packageName)
+
+        package = self.currentPackage
+        package.close()
+
+        del self.packageStack[-1]
+        if self.packageStack:
+            self.currentPackage = self.packageStack[-1]
+            self.currentPackage.freezer.excludeFrom(package.freezer)
+        else:
+            self.currentPackage = None
+
+    def module(self, moduleName, newName = None):
+        """ Adds the indicated Python module to the current package. """
+
+        if not self.currentPackage:
+            raise OutsideOfPackageError
+
+        self.currentPackage.freezer.addModule(moduleName, newName = newName)
+
+    def freeze(self, filename, compileToExe = False):
+        """ Freezes all of the current Python code into either an
+        executable (if compileToExe is true) or a dynamic library (if
+        it is false).  The resulting compiled binary is added to the
+        current package under the indicated filename.  The filename
+        should not include an extension; that will be added. """
+
+        if not self.currentPackage:
+            raise OutsideOfPackageError
+
+        package = self.currentPackage
+        freezer = package.freezer
+        freezer.done()
+
+        dirname = ''
+        basename = filename
+        if '/' in basename:
+            dirname, basename = filename.rsplit('/', 1)
+            dirname += '/'
+        basename = freezer.generateCode(basename, compileToExe = compileToExe)
+
+        package.files.append(self.PackFile(basename, newName = dirname + basename, deleteTemp = True))
+
+        # Reset the freezer for more Python files.
+        freezer.reset()
+
+
+    def file(self, filename, newNameOrDir = None):
+        """ Adds the indicated arbitrary file to the current package.
+
+        The file is placed in the named directory, or the toplevel
+        directory if no directory is specified.
+
+        The filename may include environment variable references and
+        shell globbing characters.
+
+        Certain special behavior is invoked based on the filename
+        extension.  For instance, .py files may be automatically
+        compiled and stored as Python modules.
+
+        If newNameOrDir ends in a slash character, it specifies the
+        directory in which the file should be placed.  In this case,
+        all files matched by the filename expression are placed in the
+        named directory.
+
+        If newNameOrDir ends in something other than a slash
+        character, it specifies a new filename.  In this case, the
+        filename expression must match only one file.
+
+        If newNameOrDir is unspecified or None, the file is placed in
+        the toplevel directory, regardless of its source directory.
+        
+        """
+
+        if not self.currentPackage:
+            raise OutsideOfPackageError
+
+        expanded = Filename.expandFrom(filename)
+        files = glob.glob(expanded.toOsSpecific())
+        if not files:
+            self.notify.warning("No such file: %s" % (expanded))
+            return
+
+        newName = None
+        prefix = ''
+        
+        if newNameOrDir:
+            if newNameOrDir[-1] == '/':
+                prefix = newNameOrDir
+            else:
+                newName = newNameOrDir
+                if len(files) != 1:
+                    message = 'Cannot install multiple files on target filename %s' % (newName)
+                    raise PackagerError, message
+
+        package = self.currentPackage
+
+        for filename in files:
+            filename = Filename.fromOsSpecific(filename)
+            basename = filename.getBasename()
+            if newName:
+                self.addFile(filename, newName = newName)
+            else:
+                self.addFile(filename, newName = prefix + basename)
+
+    def addFile(self, filename, newName = None):
+        """ Adds the named file, giving it the indicated name within
+        the package. """
+
+        if not self.currentPackage:
+            raise OutsideOfPackageError
+
+        self.currentPackage.files.append(
+            self.PackFile(filename, newName = newName))

direct/src/showutil/pfreeze.py

@@ -106,13 +106,17 @@ if startfile.endswith('.py') or startfile.endswith('.pyw') or \
    startfile.endswith('.pyc') or startfile.endswith('.pyo'):
     startfile = os.path.splitext(startfile)[0]
 
-freezer.addModule(startfile)
-if outputType != 'dll':
-    freezer.setMain(startfile)
+compileToExe = False
+if outputType == 'dll':
+    freezer.addModule(startfile)
+else:
+    freezer.addModule(startfile, newName = '__main__')
+    compileToExe = True
+
 freezer.done()
 
 if outputType == 'mf':
     freezer.writeMultifile(basename)
 else:
-    freezer.generateCode(basename)
+    freezer.generateCode(basename, compileToExe = compileToExe)
 

direct/src/showutil/ppackage.py

@@ -0,0 +1,258 @@
+#! /usr/bin/env python
+
+"""
+This script can be used to produce a downloadable "Package", which may
+contain arbitrary files--for instance, Python code, bam files, and/or
+compiled DLL's--and which may be downloaded by application code to
+extend an application at runtime.
+
+In addition to building the package in the first place, this script
+can also be used to generate downloadable patches of the package
+against previous versions, and manage the whole tree of patches in a
+directory structure suitable for hosting on a web server somewhere.
+
+This script is actually a wrapper around Panda's Packager.py.
+
+Usage:
+
+  ppackage.py [opts] package.pdef command
+
+Required:
+
+  package.pdef
+    The config file that describes the contents of the package file(s)
+    to be built, in excruciating detail.  Use "ppackage.py -H" to
+    describe the syntax of this file.
+
+  command
+    The action to perform.  The following commands are supported:
+
+    "build": Builds the package file(s) named in the package.pdef, and
+        places the result in the install_dir.  Does not attempt to
+        generate any patches.
+
+    "publish": Builds a package file, as above, and then generates
+        patches against previous builds, and updates install_dir and
+        persist_dir appropriately.  Instead of the current directory,
+        the built package file(s) are placed in the install_dir where
+        they may be downloaded.
+
+Options:
+
+  -i install_dir
+     The full path to a local directory to copy the
+     ready-to-be-published files into.  This directory structure is
+     populated by the "publish" command, and it may contain multiple
+     different packages from multiple different invocations of the
+     "publish" command.  It is the user's responsibility to copy this
+     directory structure to a web host where it may be downloaded by
+     the client.
+
+  -d persist_dir
+     The full path to a local directory that retains persistant state
+     between publishes.  This directory structure keeps files that are
+     used to build patches for future releases.  You should keep this
+     directory structure around for as long as you plan to support
+     this package.  If this directory structure does not exist or is
+     empty, patches will not be created for this publish; but the
+     directory structure will be populated for the next publish.
+
+  -p platform
+     Specify the platform to masquerade as.  The default is whatever
+     platform Panda has been built for.  It is probably unwise to set
+     this, unless you know what you are doing.
+
+  -H
+     Describe the syntax of the package.pdef input file.
+
+  -h
+     Display this help
+"""
+
+
+#
+# package.pdef syntax:
+#
+#   multifile <mfname> <phase>
+#
+#     Begins a new multifile.  All files named after this line and
+#     until the next multifile line will be a part of this multifile.
+#
+#     <mfname>
+#       The filename of the multifile, no directory.
+#
+#     <phase>
+#       The numeric phase in which this multifile should be downloaded.
+#
+#
+#   file <extractFlag> <filename> <dirname> <platforms>
+#
+#     Adds a single file to the current multifile.
+#
+#     <extractFlag>
+#       One of:
+#         0 - Leave this file within the multifile; it can be read by
+#             Panda from there.
+#         1 - Extract this file from the multifile, but do not bother
+#             to hash check it on restart.
+#         2 - Extract this file, and hash check it every time the
+#             client starts, to ensure it is not changed.
+#
+#     <filename>
+#       The name of the file to add.  This is the full path to the
+#       file on the publishing machine at the time this script is run.
+#
+#     <dirname>
+#       The directory in which to install the file, on the client.
+#       This should be a relative pathname from the game directory.
+#       The file is written to the multifile with its directory part
+#       taken from this, and its basename taken from the source
+#       filename, above.  Also, if the file is extracted, it will be
+#       written into this directory on the client machine.
+#
+#       The directory name "toplevel" is treated as a special case;
+#       this maps to the game directory itself, and is used for files
+#       in the initial download.
+#
+#     <platforms>
+#       A comma-delimited list of platforms for which this file should
+#       be included with the distribution.  Presently, the only
+#       options are WIN32 and/or OSX.
+#
+#
+#   dir <extractFlag> <localDirname> <dirname>
+#
+#     Adds an entire directory tree to the current multifile.  The
+#     named directory is searched recursively and all files found are
+#     added to the current multifile, as if they were listed one a
+#     time with a file command.
+#
+#     <extractFlag>
+#       (Same as for the file command, above.)
+#
+#     <localDirname>
+#       The name of the local directory to scan on the publishing
+#       machine.
+#
+#     <dirname>
+#       The name of the corresponding local directory on the client
+#       machine; similar to <dirname> in the file command, above.
+#
+#
+#   module modulename
+#
+#     Adds the named Python module to the exe or dll archive.  All files
+#     named by module, until the next freeze_exe or freeze_dll command (below),
+#     will be compiled and placed into the same archive.  
+#
+#   exclude_module modulename
+#
+#     Excludes the named Python module from the archive.  This module
+#     will not be included in the archive, but an import command may
+#     still find it if a .py file exists on disk.
+#
+#   forbid_module modulename
+#
+#     Excludes the named Python module from the archive.  This module
+#     will specifically never be imported by the resulting executable,
+#     even if a .py file exists on disk.  (However, a module command
+#     appearing in a later phase--e.g. Phase3.pyd--can override an
+#     earlier forbid_module command.)
+#
+#   dc_module file.dc
+#
+#     Adds the modules imported by the indicated dc file to the
+#     archive.  Normally this is not necessary if the file.dc is
+#     explicitly included in the package.pdef; but this command may be
+#     useful if the file.dc is imported in a later phase.
+#
+#   freeze_exe <extractFlag> <exeFilename> <mainModule> <dirname>
+#
+#     <extractFlag>
+#       (Same as for the file command, above.)
+#
+#     <exeFilename>
+#       The name of the executable file to generate.  Do not include
+#       an extension name; on Windows, the default extension is .exe;
+#       on OSX there is no extension.
+#
+#     <mainModule>
+#       The name of the python module that will be invoked first (like
+#       main()) when the resulting executable is started.
+#
+#     <dirname>
+#       (Same as for the file command, above.)
+#
+#   freeze_dll <extractFlag> <dllFilename> <dirname>
+#
+#     <extractFlag>
+#       (Same as for the file command, above.)
+#
+#     <dllFilename>
+#       The name of the shared library file to generate.  Do not include
+#       an extension name; on Windows, the default extension is .pyd;
+#       on OSX the extension is .so.
+#
+#     <dirname>
+#       (Same as for the file command, above.)
+
+import sys
+import getopt
+import os
+
+from direct.showutil import Packager
+from pandac.PandaModules import *
+
+def usage(code, msg = ''):
+    print >> sys.stderr, __doc__
+    print >> sys.stderr, msg
+    sys.exit(code)
+
+packager = Packager.Packager()
+
+try:
+    opts, args = getopt.getopt(sys.argv[1:], 'i:d:p:Hh')
+except getopt.error, msg:
+    usage(1, msg)
+
+for opt, arg in opts:
+    if opt == '-i':
+        packager.installDir = Filename.fromOsSpecific(arg)
+    elif opt == '-d':
+        packager.persistDir = Filename.fromOsSpecific(arg)
+    elif opt == '-p':
+        packager.platform = arg
+        
+    elif opt == '-h':
+        usage(0)
+    elif opt == '-H':
+        print 'Not yet implemented.'
+        sys.exit(1)
+    else:
+        print 'illegal option: ' + flag
+        sys.exit(1)
+
+if not args:
+    usage(0)
+    
+if len(args) != 2:
+    usage(1)
+
+packageDef = Filename.fromOsSpecific(args[0])
+command = args[1]
+
+
+if command == 'build':
+    #packager.doBuild()
+    if not packager.persistDir:
+        packager.persistDir = Filename('.')
+    packager.setup()
+    packager.readPackageDef(packageDef)
+elif command == 'publish':
+    packager.setup()
+    packager.doPublish()
+else:
+    print 'Undefined command: ' + command
+    sys.exit(1)
+    
+