integrate make_contents into Packager

This commit is contained in:
David Rose 2009-08-24 18:42:47 +00:00
parent f4e14c8455
commit bfb948c5a8
6 changed files with 209 additions and 382 deletions

View File

@ -1,5 +1,5 @@
import os
from pandac.PandaModules import Filename, HashVal
from pandac.PandaModules import Filename, HashVal, VirtualFileSystem
class FileSpec:
""" This class represents a disk file whose hash and size
@ -7,7 +7,31 @@ class FileSpec:
verify whether the file on disk matches the version demanded by
the xml. """
def __init__(self, xelement):
def __init__(self):
pass
def fromFile(self, packageDir, filename):
""" Reads the file information from the indicated file. """
vfs = VirtualFileSystem.getGlobalPtr()
filename = Filename(filename)
pathname = Filename(packageDir, filename)
self.filename = filename.cStr()
self.basename = filename.getBasename()
st = os.stat(pathname.toOsSpecific())
self.size = st.st_size
self.timestamp = st.st_mtime
hv = HashVal()
hv.hashFile(pathname)
self.hash = hv.asHex()
def loadXml(self, xelement):
""" Reads the file information from the indicated XML
element. """
self.filename = xelement.Attribute('filename')
self.basename = Filename(self.filename).getBasename()
size = xelement.Attribute('size')
@ -24,6 +48,15 @@ class FileSpec:
self.hash = xelement.Attribute('hash')
def storeXml(self, xelement):
""" Adds the file information to the indicated XML
element. """
xelement.SetAttribute('filename', self.filename)
xelement.SetAttribute('size', str(self.size))
xelement.SetAttribute('timestamp', str(self.timestamp))
xelement.SetAttribute('hash', self.hash)
def quickVerify(self, packageDir = None, pathname = None):
""" Performs a quick test to ensure the file has not been
modified. This test is vulnerable to people maliciously

View File

@ -58,7 +58,8 @@ class HostInfo:
platform = xpackage.Attribute('platform')
version = xpackage.Attribute('version')
package = self.__makePackage(name, platform, version)
package.descFile = FileSpec(xpackage)
package.descFile = FileSpec()
package.descFile.loadXml(xpackage)
xpackage = xpackage.NextSiblingElement('package')
@ -69,7 +70,8 @@ class HostInfo:
platform = ximport.Attribute('platform')
version = ximport.Attribute('version')
package = self.__makePackage(name, platform, version)
package.importDescFile = FileSpec(ximport)
package.importDescFile = FileSpec()
package.importDescFile.loadXml(ximport)
ximport = ximport.NextSiblingElement('import')

View File

@ -10,6 +10,7 @@ import marshal
import new
import string
import types
from direct.p3d.FileSpec import FileSpec
from direct.showbase import Loader
from direct.showbase import AppRunnerGlobal
from direct.showutil import FreezeTool
@ -130,6 +131,56 @@ class Packager:
else:
return self.glob.matches(filename.cStr())
class PackageEntry:
""" This corresponds to an entry in the contents.xml file. """
def __init__(self):
pass
def getKey(self):
""" Returns a tuple used for sorting the PackageEntry
objects uniquely per package. """
return (self.packageName, self.platform, self.version)
def fromFile(self, packageName, platform, version, solo, isImport,
installDir, descFilename):
self.packageName = packageName
self.platform = platform
self.version = version
self.solo = solo
self.isImport = isImport
self.descFile = FileSpec()
self.descFile.fromFile(installDir, descFilename)
def loadXml(self, xelement):
self.packageName = xelement.Attribute('name')
self.platform = xelement.Attribute('platform')
self.version = xelement.Attribute('version')
solo = xelement.Attribute('solo')
self.solo = int(solo or '0')
self.isImport = (xelement.Value() == 'import')
self.descFile = FileSpec()
self.descFile.loadXml(xelement)
def makeXml(self):
""" Returns a new TiXmlElement. """
value = 'package'
if self.isImport:
value = 'import'
xelement = TiXmlElement(value)
xelement.SetAttribute('name', self.packageName)
if self.platform:
xelement.SetAttribute('platform', self.platform)
if self.version:
xelement.SetAttribute('version', self.version)
if self.solo:
xelement.SetAttribute('solo', '1')
self.descFile.storeXml(xelement)
return xelement
class Package:
def __init__(self, packageName, packager):
self.packageName = packageName
@ -408,13 +459,26 @@ class Packager:
self.writeDescFile()
self.writeImportDescFile()
# Replace or add the entry in the contents.
a = Packager.PackageEntry()
a.fromFile(self.packageName, self.platform, self.version,
False, False, self.packager.installDir,
self.packageDesc)
b = Packager.PackageEntry()
b.fromFile(self.packageName, self.platform, self.version,
False, True, self.packager.installDir,
self.packageImportDesc)
self.packager.contents[a.getKey()] = [a, b]
self.packager.contentsChanged = True
self.cleanup()
def installSolo(self):
""" Installs the package as a "solo", which means we
simply copy all files into the install directory. This is
primarily intended for the "coreapi" plugin, which is just
a single dll and a jpg file; but it can support other
simply copy the one file into the install directory. This
is primarily intended for the "coreapi" plugin, which is
just a single dll and a jpg file; but it can support other
kinds of similar "solo" packages as well. """
packageDir = self.packageName
@ -430,16 +494,23 @@ class Packager:
for origFile in origFiles:
origFile.getFilename().unlink()
if not self.files:
# No files, never mind.
return
Filename(installPath, '').makeDir()
files = []
for file in self.files:
if file.isExcluded(self):
# Skip this file.
continue
files.append(file)
if not files:
# No files, never mind.
return
if len(files) != 1:
raise PackagerError, 'Multiple files in "solo" package %s' % (self.packageName)
Filename(installPath, '').makeDir()
file = files[0]
targetPath = Filename(installPath, file.newName)
targetPath.setBinary()
file.filename.setBinary()
@ -447,6 +518,15 @@ class Packager:
print "Could not copy %s to %s" % (
file.filename, targetPath)
# Replace or add the entry in the contents.
a = Packager.PackageEntry()
a.fromFile(self.packageName, self.platform, self.version,
True, False, self.packager.installDir,
Filename(packageDir, file.newName))
self.packager.contents[a.getKey()] = [a]
self.packager.contentsChanged = True
self.cleanup()
def cleanup(self):
@ -1106,6 +1186,7 @@ class Packager:
for moduleName, mdef in p2.moduleNames.items():
self.skipModules[moduleName] = mdef
# Packager constructor
def __init__(self):
# The following are config settings that the caller may adjust
@ -1268,6 +1349,10 @@ class Packager:
# A table of all known packages by name.
self.packages = {}
# A list of PackageEntry objects read from the contents.xml
# file.
self.contents = {}
def addWindowsSearchPath(self, searchPath, varname):
""" Expands $varname, interpreting as a Windows-style search
path, and adds its contents to the indicated DSearchPath. """
@ -1303,133 +1388,13 @@ class Packager:
if not PandaSystem.getPackageVersionString() or not PandaSystem.getPackageHostUrl():
raise PackagerError, 'This script must be run using a version of Panda3D that has been built\nfor distribution. Try using ppackage.p3d or packp3d.p3d instead.'
def __expandVariable(self, line, p):
""" Given that line[p] is a dollar sign beginning a variable
reference, advances p to the first dollar sign following the
reference, and looks up the variable referenced.
self.readContentsFile()
Returns (value, p) where value is the value of the named
variable, and p is the first character following the variable
reference. """
def close(self):
""" Called after reading all of the package def files, this
performs any final cleanup appropriate. """
p += 1
if p >= len(line):
return '', p
var = ''
if line[p] == '{':
# Curly braces exactly delimit the variable name.
p += 1
while p < len(line) and line[p] != '}':
var += line[p]
p += 1
else:
# Otherwise, a string of alphanumeric characters,
# including underscore, delimits the variable name.
var += line[p]
p += 1
while p < len(line) and (line[p] in string.letters or line[p] in string.digits or line[p] == '_'):
var += line[p]
p += 1
return ExecutionEnvironment.getEnvironmentVariable(var), p
def __splitLine(self, line):
""" Separates the indicated line into words at whitespace.
Quotation marks and escape characters protect spaces. This is
designed to be similar to the algorithm employed by the Unix
shell. """
words = []
p = 0
while p < len(line):
if line[p] == '#':
# A word that begins with a hash mark indicates an
# inline comment, and the end of the parsing.
break
# Scan to the end of the word.
word = ''
while p < len(line) and line[p] not in string.whitespace:
if line[p] == '\\':
# Output an escaped character.
p += 1
if p < len(line):
word += line[p]
p += 1
elif line[p] == '$':
# Output a variable reference.
expand, p = self.__expandVariable(line, p)
word += expand
elif line[p] == '"':
# Output a double-quoted string.
p += 1
while p < len(line) and line[p] != '"':
if line[p] == '\\':
# Output an escaped character.
p += 1
if p < len(line):
word += line[p]
p += 1
elif line[p] == '$':
# Output a variable reference.
expand, p = self.__expandVariable(line, p)
word += expand
else:
word += line[p]
p += 1
elif line[p] == "'":
# Output a single-quoted string. Escape
# characters and dollar signs within single quotes
# are not special.
p += 1
while p < len(line) and line[p] != "'":
word += line[p]
p += 1
else:
# Output a single character.
word += line[p]
p += 1
words.append(word)
# Scan to the beginning of the next word.
while p < len(line) and line[p] in string.whitespace:
p += 1
return words
def __getNextLine(self):
""" Extracts the next line from self.inFile, and splits it
into words. Returns the list of words, or None at end of
file. """
line = self.inFile.readline()
self.lineNum += 1
while line:
line = line.strip()
if not line:
# Skip the line, it was just a blank line
pass
elif line[0] == '#':
# Eat python-style comments.
pass
else:
return self.__splitLine(line)
line = self.inFile.readline()
self.lineNum += 1
# End of file.
return None
self.writeContentsFile()
def readPackageDef(self, packageDef):
""" Reads the named .pdef file and constructs the packages
@ -1568,7 +1533,7 @@ class Packager:
self.host = hostUrl
# The descriptive name, if specified, is kept until the end,
# where it may be passed to make_contents by ppackage.py.
# where it may be written into the contents file.
if descriptiveName:
self.hostDescriptiveName = descriptiveName
@ -2188,6 +2153,59 @@ class Packager:
explicit = False)
def readContentsFile(self):
""" Reads the contents.xml file at the beginning of
processing. """
self.contents = {}
self.contentsChanged = False
contentsFilename = Filename(self.installDir, 'contents.xml')
doc = TiXmlDocument(contentsFilename.toOsSpecific())
if not doc.LoadFile():
# Couldn't read file.
return
xcontents = doc.FirstChildElement('contents')
if xcontents:
if self.hostDescriptiveName is None:
self.hostDescriptiveName = xcontents.Attribute('descriptive_name')
xelement = xcontents.FirstChildElement()
while xelement:
package = self.PackageEntry()
package.loadXml(xelement)
self.contents.setdefault(package.getKey(), []).append(package)
xelement = xelement.NextSiblingElement()
def writeContentsFile(self):
""" Rewrites the contents.xml file at the end of
processing. """
if not self.contentsChanged:
# No need to rewrite.
return
contentsFilename = Filename(self.installDir, 'contents.xml')
doc = TiXmlDocument(contentsFilename.toOsSpecific())
decl = TiXmlDeclaration("1.0", "utf-8", "")
doc.InsertEndChild(decl)
xcontents = TiXmlElement('contents')
if self.hostDescriptiveName:
xcontents.SetAttribute('descriptive_name', self.hostDescriptiveName)
contents = self.contents.items()
contents.sort()
for key, entryList in contents:
for entry in entryList:
xelement = entry.makeXml()
xcontents.InsertEndChild(xelement)
doc.InsertEndChild(xcontents)
doc.SaveFile()
# The following class and function definitions represent a few sneaky
# Python tricks to allow the pdef syntax to contain the pseudo-Python
# code they do. These tricks bind the function and class definitions

View File

@ -1,215 +0,0 @@
#! /usr/bin/env python
"""
This command will build the contents.xml file at the top of a Panda3D
download hierarchy. This file lists all of the packages hosted here,
along with their current versions.
This program runs on a local copy of the hosting directory hierarchy;
it must be a complete copy to generate a complete contents.xml file.
make_contents.py [opts]
Options:
-i install_dir
The full path to a local directory that contains the
ready-to-be-published files, as populated by one or more
iterations of the ppackage script. It is the user's
responsibility to copy this directory structure to a server.
-n "host descriptive name"
Specifies a descriptive name of the download server that will
host these contents. This name may be presented to the user when
managing installed packages. If this option is omitted, the name
is unchanged from the previous pass.
"""
import sys
import getopt
import os
import types
try:
import hashlib
except ImportError:
# Legacy Python support
import md5 as hashlib
class ArgumentError(AttributeError):
pass
class FileSpec:
""" Represents a single file in the directory, and its associated
timestamp, size, and md5 hash. """
def __init__(self, filename, pathname):
self.filename = filename
self.pathname = pathname
s = os.stat(pathname)
self.size = s.st_size
self.timestamp = int(s.st_mtime)
m = hashlib.md5()
m.update(open(pathname, 'rb').read())
self.hash = m.hexdigest()
def getParams(self):
return 'filename="%s" size="%s" timestamp="%s" hash="%s"' % (
self.filename, self.size, self.timestamp, self.hash)
class ContentsMaker:
def __init__(self):
self.installDir = None
self.hostDescriptiveName = None
def build(self):
if not self.installDir:
raise ArgumentError, "Stage directory not specified."
self.packages = []
self.scanDirectory()
if not self.packages:
raise ArgumentError, "No packages found."
contentsFileBasename = 'contents.xml'
contentsFilePathname = os.path.join(self.installDir, contentsFileBasename)
contentsLine = None
if self.hostDescriptiveName is not None:
if self.hostDescriptiveName:
contentsLine = '<contents descriptive_name="%s">' % (
self.quoteString(self.hostDescriptiveName))
else:
contentsLine = self.readContentsLine(contentsFilePathname)
if not contentsLine:
contentsLine = '<contents>'
# Now write the contents.xml file.
f = open(contentsFilePathname, 'w')
print >> f, '<?xml version="1.0" encoding="utf-8" ?>'
print >> f, ''
print >> f, contentsLine
for type, packageName, packagePlatform, packageVersion, file, solo in self.packages:
extra = ''
if solo:
extra += 'solo="1" '
print >> f, ' <%s name="%s" platform="%s" version="%s" %s%s />' % (
type, packageName, packagePlatform or '', packageVersion or '', extra, file.getParams())
print >> f, '</contents>'
f.close()
def readContentsLine(self, contentsFilePathname):
""" Reads the previous iteration of contents.xml to get the
previous top-level contents line, which contains the
hostDescriptiveName. """
try:
f = open(contentsFilePathname, 'r')
except IOError:
return None
for line in f.readlines():
if line.startswith('<contents'):
return line.rstrip()
return None
def quoteString(self, str):
""" Correctly quotes a string for embedding in the xml file. """
if isinstance(str, types.UnicodeType):
str = str.encode('utf-8')
str = str.replace('&', '&amp;')
str = str.replace('"', '&quot;')
str = str.replace('\'', '&apos;')
str = str.replace('<', '&lt;')
str = str.replace('>', '&gt;')
return str
def scanDirectory(self):
""" Walks through all the files in the stage directory and
looks for the package directory xml files. """
startDir = self.installDir
if startDir.endswith(os.sep):
startDir = startDir[:-1]
prefix = startDir + os.sep
for dirpath, dirnames, filenames in os.walk(startDir):
if dirpath == startDir:
localpath = ''
xml = ''
else:
assert dirpath.startswith(prefix)
localpath = dirpath[len(prefix):].replace(os.sep, '/') + '/'
xml = dirpath[len(prefix):].replace(os.sep, '_') + '.xml'
solo = False
# A special case: if a directory contains just one file,
# it's a "solo", not an xml package.
if len(filenames) == 1 and not filenames[0].endswith('.xml'):
xml = filenames[0]
solo = True
if xml not in filenames:
continue
if localpath.count('/') == 1:
packageName, junk = localpath.split('/')
packageVersion = None
packagePlatform = None
elif localpath.count('/') == 2:
packageName, packageVersion, junk = localpath.split('/')
packagePlatform = None
elif localpath.count('/') == 3:
packageName, packagePlatform, packageVersion, junk = localpath.split('/')
else:
continue
file = FileSpec(localpath + xml,
os.path.join(self.installDir, localpath + xml))
print file.filename
self.packages.append(('package', packageName, packagePlatform, packageVersion, file, solo))
if not solo:
# Look for an _import.xml file, too.
xml = xml[:-4] + '_import.xml'
try:
file = FileSpec(localpath + xml,
os.path.join(self.installDir, localpath + xml))
except OSError:
file = None
if file:
print file.filename
self.packages.append(('import', packageName, packagePlatform, packageVersion, file, False))
def makeContents(args):
opts, args = getopt.getopt(args, 'i:n:h')
cm = ContentsMaker()
cm.installDir = '.'
for option, value in opts:
if option == '-i':
cm.installDir = value
elif option == '-n':
cm.hostDescriptiveName = value
elif option == '-h':
print __doc__
sys.exit(1)
cm.build()
if __name__ == '__main__':
try:
makeContents(sys.argv[1:])
except ArgumentError, e:
print e.args[0]
sys.exit(1)

View File

@ -87,7 +87,6 @@ import getopt
import os
from direct.p3d import Packager
from direct.p3d import make_contents
from pandac.PandaModules import *
def usage(code, msg = ''):
@ -112,9 +111,9 @@ for opt, arg in opts:
elif opt == '-p':
packager.platform = arg
elif opt == '-u':
package.host = arg
packager.host = arg
elif opt == '-n':
package.hostDescriptiveName = arg
packager.hostDescriptiveName = arg
elif opt == '-h':
usage(0)
@ -140,26 +139,10 @@ packager.installSearch.prependDirectory(packager.installDir)
try:
packager.setup()
packages = packager.readPackageDef(packageDef)
packager.close()
except Packager.PackagerError:
# Just print the error message and exit gracefully.
inst = sys.exc_info()[1]
print inst.args[0]
#raise
sys.exit(1)
# Look to see if we built any true packages, or if all of them were
# p3d files.
anyPackages = False
for package in packages:
if not package.p3dApplication:
anyPackages = True
break
if anyPackages:
# If we built any true packages, then update the contents.xml at
# the root of the install directory.
cm = make_contents.ContentsMaker()
cm.installDir = packager.installDir.toOsSpecific()
cm.hostDescriptiveName = packager.hostDescriptiveName
cm.build()

View File

@ -163,7 +163,10 @@ get_package_desc_file(FileSpec &desc_file, // out
const char *platform = xpackage->Attribute("platform");
const char *version = xpackage->Attribute("version");
const char *solo = xpackage->Attribute("solo");
if (name != NULL && platform != NULL && version != NULL &&
if (version == NULL) {
version = "";
}
if (name != NULL && platform != NULL &&
package_name == name &&
inst_mgr->get_platform() == platform &&
package_version == version) {
@ -190,7 +193,10 @@ get_package_desc_file(FileSpec &desc_file, // out
if (platform == NULL) {
platform = "";
}
if (name != NULL && version != NULL &&
if (version == NULL) {
version = "";
}
if (name != NULL &&
package_name == name &&
*platform == '\0' &&
package_version == version) {