xml, compression

direct/metalibs/direct/Sources.pp

@@ -9,7 +9,7 @@
 #define USE_PACKAGES native_net
 
 #define COMPONENT_LIBS \
-  directbase dcparser showbase deadrec directd interval distributed motiontrail http
+  directbase dcparser showbase deadrec directd interval distributed motiontrail http dxml
 
 #define OTHER_LIBS \
   panda:m \

direct/src/dxml/Sources.pp

@@ -0,0 +1,15 @@
+#begin lib_target
+  #define BUILD_TARGET $[HAVE_TINYXML]
+  #define USE_PACKAGES tinyxml
+
+  #define TARGET dxml
+  #define LOCAL_LIBS directbase
+  #define OTHER_LIBS \
+    interrogatedb:c dconfig:c dtoolconfig:m \
+    dtoolutil:c dtoolbase:c dtool:m
+
+  #define SOURCES \
+    config_dxml.h config_dxml.cxx
+
+  #define IGATESCAN all
+#end lib_target

direct/src/dxml/config_dxml.N

@@ -0,0 +1,14 @@
+forcetype TiXmlAttribute
+forcetype TiXmlBase
+forcetype TiXmlComment
+forcetype TiXmlDeclaration
+forcetype TiXmlDocument
+forcetype TiXmlElement
+forcetype TiXmlNode
+forcetype TiXmlParsingData
+forcetype TiXmlText
+forcetype TiXmlUnknown
+forcetype TiXmlVisitor
+
+# This method can't be exported, because it steals ownership of the pointer.
+ignoremember LinkEndChild

direct/src/dxml/config_dxml.cxx

@@ -0,0 +1,44 @@
+// Filename: config_dxml.cxx
+// Created by: drose (08Aug09)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+#include "config_dxml.h"
+#include "dconfig.h"
+
+BEGIN_PUBLISH
+#include "tinyxml.h"
+END_PUBLISH
+
+Configure(config_dxml);
+NotifyCategoryDef(dxml, "");
+
+ConfigureFn(config_dxml) {
+  init_libdxml();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: init_libdxml
+//  Description: Initializes the library.  This must be called at
+//               least once before any of the functions or classes in
+//               this library can be used.  Normally it will be
+//               called by the static initializers and need not be
+//               called explicitly, but special cases exist.
+////////////////////////////////////////////////////////////////////
+void
+init_libdxml() {
+  static bool initialized = false;
+  if (initialized) {
+    return;
+  }
+  initialized = true;
+}

direct/src/dxml/config_dxml.h

@@ -0,0 +1,36 @@
+// Filename: config_dxml.h
+// Created by: drose (08Aug09)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+#ifndef CONFIG_DXML_H
+#define CONFIG_DXML_H
+
+#include "directbase.h"
+#include "notifyCategoryProxy.h"
+#include "dconfig.h"
+
+// The purpose of this directory is to expose tinyxml interfaces to
+// Python via interrogate.
+
+// tinyxml.h requires having the symbol TIXML_USE_STL already defined
+// before you include it.
+
+#ifndef TIXML_USE_STL
+#define TIXML_USE_STL
+#endif
+
+NotifyCategoryDecl(dxml, EXPCL_DIRECT, EXPTP_DIRECT);
+
+extern EXPCL_DIRECT void init_libdxml();
+
+#endif

direct/src/plugin/make_package.py

@@ -5,7 +5,7 @@ This command is used to build a downloadable package for the p3d
 plugin to retrieve and install.  It examines the files in the current
 directory, assumes they are all intended to be part of the package,
 and constructs the necessary package xml file and archive file for
-hosting on the web serevr.
+hosting on the web server.
 
 make_package.py [opts]
 

direct/src/showutil/FreezeTool.py

@@ -813,14 +813,14 @@ class Freezer:
                 multifile.removeSubfile(filename + '.pyo')
 
         # Attempt to add the original source file if we can.
-        if self.storePythonSource:
+        sourceFilename = None
-            sourceFilename = None
+        if mdef.filename and mdef.filename.getExtension() == "py":
-            if mdef.filename and mdef.filename.getExtension() == "py":
+            sourceFilename = mdef.filename
-                sourceFilename = mdef.filename
+        elif getattr(module, '__file__', None):
-            elif getattr(module, '__file__', None):
+            sourceFilename = Filename.fromOsSpecific(module.__file__)
-                sourceFilename = Filename.fromOsSpecific(module.__file__)
+            sourceFilename.setExtension("py")
-                sourceFilename.setExtension("py")
             
+        if self.storePythonSource:
             if sourceFilename and sourceFilename.exists():
                 filename += '.py'
                 multifile.addSubfile(filename, sourceFilename, 0)
@@ -832,14 +832,13 @@ class Freezer:
         else:
             filename += '.pyo'
 
+        code = None
         if module:
             # Get the compiled code directly from the module object.
             code = getattr(module, "__code__", None)
         else:
             # Read the code from the source file and compile it on-the-fly.
-            sourceFilename = None
+            if sourceFilename and sourceFilename.exists():
-            if mdef.filename and mdef.filename.getExtension() == "py":
-                sourceFilename = mdef.filename
                 source = open(sourceFilename.toOsSpecific(), 'r').read()
                 if source and source[-1] != '\n':
                     source = source + '\n'

direct/src/showutil/Packager.py

@@ -61,19 +61,32 @@ class Packager:
         def close(self):
             """ Writes out the contents of the current package. """
 
-            packageFilename = self.packageName
+            self.packageBasename = self.packageName
+            packageDir = self.packageName
             if self.platform:
-                packageFilename += '_' + self.platform
+                self.packageBasename += '_' + self.platform
+                packageDir += '/' + self.platform
             if self.version:
-                packageFilename += '_' + self.version
+                self.packageBasename += '_' + self.version
+                packageDir += '/' + self.version
 
+            self.packageDesc = self.packageBasename + '.xml'
+            self.packageImportDesc = self.packageBasename + '_import.xml'
             if self.p3dApplication:
-                packageFilename += '.p3d'
+                self.packageBasename += '.p3d'
+                packageDir = ''
             else:
-                packageFilename += '.mf'
+                self.packageBasename += '.mf'
-            
+                packageDir += '/'
+
+            self.packageFilename = packageDir + self.packageBasename
+            self.packageDesc = packageDir + self.packageDesc
+            self.packageImportDesc = packageDir + self.packageImportDesc
+
+            Filename(self.packageFilename).makeDir()
+
             try:
-                os.unlink(packageFilename)
+                os.unlink(self.packageFilename)
             except OSError:
                 pass
 
@@ -81,7 +94,10 @@ class Packager:
                 self.multifile = None
             else:
                 self.multifile = Multifile()
-                self.multifile.openReadWrite(packageFilename)
+                self.multifile.openReadWrite(self.packageFilename)
+
+            self.extracts = []
+            self.components = []
 
             # Exclude modules already imported in a required package.
             for moduleName in self.skipModules.keys():
@@ -131,12 +147,9 @@ class Packager:
                         self.addBamFile(file)
                     elif ext in self.packager.imageExtensions:
                         self.addTexture(file)
-                    elif ext in self.packager.uncompressibleExtensions:
-                        # An uncompressible file.
-                        self.multifile.addSubfile(file.newName, file.filename, 0)
                     else:
                         # Any other file.
-                        self.multifile.addSubfile(file.newName, file.filename, self.compressionLevel)
+                        self.addComponent(file)
 
             # Pick up any unfrozen Python files.
             self.freezer.done()
@@ -146,17 +159,101 @@ class Packager:
             for moduleName in self.freezer.getAllModuleNames():
                 self.moduleNames[moduleName] = True
 
+                xmodule = TiXmlElement('module')
+                xmodule.SetAttribute('name', moduleName)
+                self.components.append(xmodule)
+
             if not self.dryRun:
                 self.freezer.addToMultifile(self.multifile)
                 self.multifile.repack()
                 self.multifile.close()
 
+                if not self.p3dApplication:
+                    self.compressMultifile()
+                    self.writeDescFile()
+                    self.writeImportDescFile()
+
             # Now that all the files have been packed, we can delete
             # the temporary files.
             for file in self.files:
                 if file.deleteTemp:
                     file.filename.unlink()
 
+        def compressMultifile(self):
+            """ Compresses the .mf file into an .mf.pz file. """
+
+            compressedName = self.packageFilename + '.pz'
+            if not compressFile(self.packageFilename, compressedName, 6):
+                message = 'Unable to write %s' % (compressedName)
+                raise PackagerError, message
+
+        def writeDescFile(self):
+            doc = TiXmlDocument(self.packageDesc)
+            decl = TiXmlDeclaration("1.0", "utf-8", "")
+            doc.InsertEndChild(decl)
+
+            xpackage = TiXmlElement('package')
+            xpackage.SetAttribute('name', self.packageName)
+            if self.platform:
+                xpackage.SetAttribute('platform', self.platform)
+            if self.version:
+                xpackage.SetAttribute('version', self.version)
+
+            xuncompressedArchive = self.getFileSpec(
+                'uncompressed_archive', self.packageFilename, self.packageBasename)
+            xcompressedArchive = self.getFileSpec(
+                'compressed_archive', self.packageFilename + '.pz', self.packageBasename + '.pz')
+            xpackage.InsertEndChild(xuncompressedArchive)
+            xpackage.InsertEndChild(xcompressedArchive)
+
+            for xextract in self.extracts:
+                xpackage.InsertEndChild(xextract)
+
+            doc.InsertEndChild(xpackage)
+            doc.SaveFile()
+
+        def writeImportDescFile(self):
+            doc = TiXmlDocument(self.packageImportDesc)
+            decl = TiXmlDeclaration("1.0", "utf-8", "")
+            doc.InsertEndChild(decl)
+
+            xpackage = TiXmlElement('package')
+            xpackage.SetAttribute('name', self.packageName)
+            if self.platform:
+                xpackage.SetAttribute('platform', self.platform)
+            if self.version:
+                xpackage.SetAttribute('version', self.version)
+
+            for xcomponent in self.components:
+                xpackage.InsertEndChild(xcomponent)
+
+            doc.InsertEndChild(xpackage)
+            doc.SaveFile()
+
+
+        def getFileSpec(self, element, filename, newName):
+            """ Returns an xcomponent or similar element with the file
+            information for the indicated file. """
+            
+            xspec = TiXmlElement(element)
+
+            filename = Filename(filename)
+            size = filename.getFileSize()
+            timestamp = filename.getTimestamp()
+
+            hv = HashVal()
+            hv.hashFile(filename)
+            hash = hv.asHex()
+
+            xspec.SetAttribute('filename', newName)
+            xspec.SetAttribute('size', str(size))
+            xspec.SetAttribute('timestamp', str(timestamp))
+            xspec.SetAttribute('hash', hash)
+
+            return xspec
+
+            
+
         def addPyFile(self, file):
             """ Adds the indicated python file, identified by filename
             instead of by module name, to the package. """
@@ -187,7 +284,7 @@ class Packager:
 
             bamName = Filename(file.newName)
             bamName.setExtension('bam')
-            self.addNode(np.node(), bamName.cStr())
+            self.addNode(np.node(), file.filename, bamName.cStr())
 
         def addBamFile(self, file):
             # Load the bam file so we can massage its textures.
@@ -202,15 +299,15 @@ class Packager:
             if not node:
                 raise StandardError, 'Not a model file: %s' % (file.filename)
 
-            self.addNode(node, file.newName)
+            self.addNode(node, file.filename, file.newName)
 
-        def addNode(self, node, filename):
+        def addNode(self, node, filename, newName):
             """ Converts the indicated node to a bam stream, and adds the
-            bam file to the multifile under the indicated filename. """
+            bam file to the multifile under the indicated newName. """
 
             # If the Multifile already has a file by this name, don't
             # bother adding it again.
-            if self.multifile.findSubfile(filename) >= 0:
+            if self.multifile.findSubfile(newName) >= 0:
                 return
 
             # Be sure to import all of the referenced textures, and tell
@@ -219,7 +316,7 @@ class Packager:
             for tex in NodePath(node).findAllTextures():
                 if not tex.hasFullpath() and tex.hasRamImage():
                     # We need to store this texture as a raw-data image.
-                    # Clear the filename so this will happen
+                    # Clear the newName so this will happen
                     # automatically.
                     tex.clearFilename()
                     tex.clearAlphaFilename()
@@ -247,11 +344,15 @@ class Packager:
 
             # Now we have an in-memory bam file.
             stream.seekg(0)
-            self.multifile.addSubfile(filename, stream, self.compressionLevel)
+            self.multifile.addSubfile(newName, stream, self.compressionLevel)
 
             # Flush it so the data gets written to disk immediately, so we
             # don't have to keep it around in ram.
             self.multifile.flush()
+            
+            xcomponent = TiXmlElement('component')
+            xcomponent.SetAttribute('filename', newName)
+            self.components.append(xcomponent)
 
         def addFoundTexture(self, filename):
             """ Adds the newly-discovered texture to the output, if it has
@@ -293,7 +394,27 @@ class Packager:
 
             # Texture file formats are generally already compressed and
             # not further compressible.
-            self.multifile.addSubfile(file.newName, file.filename, 0)
+            self.addComponent(file, compressible = False)
+
+        def addComponent(self, file, compressible = True, extract = False):
+            ext = Filename(file.newName).getExtension()
+            if ext in self.packager.uncompressibleExtensions:
+                compressible = False
+            if ext in self.packager.extractExtensions:
+                extract = True
+                
+            compressionLevel = 0
+            if compressible:
+                compressionLevel = self.compressionLevel
+                
+            self.multifile.addSubfile(file.newName, file.filename, compressionLevel)
+            if extract:
+                xextract = self.getFileSpec('extract', file.filename, file.newName)
+                self.extracts.append(xextract)
+
+            xcomponent = TiXmlElement('component')
+            xcomponent.SetAttribute('filename', file.newName)
+            self.components.append(xcomponent)
 
     def __init__(self):
 
@@ -350,6 +471,9 @@ class Packager:
         # processing.
         self.binaryExtensions = [ 'ttf', 'wav', 'mid' ]
 
+        # Files that should be extracted to disk.
+        self.extractExtensions = [ 'dll', 'so', 'dylib', 'exe' ]
+
         # Binary files that are considered uncompressible, and are
         # copied without compression.
         self.uncompressibleExtensions = [ 'mp3', 'ogg' ]