From bfb948c5a826ef9fedf964661ba5ac9413abf6bf Mon Sep 17 00:00:00 2001 From: David Rose Date: Mon, 24 Aug 2009 18:42:47 +0000 Subject: [PATCH] integrate make_contents into Packager --- direct/src/p3d/FileSpec.py | 37 +++- direct/src/p3d/HostInfo.py | 6 +- direct/src/p3d/Packager.py | 300 +++++++++++++++++--------------- direct/src/p3d/make_contents.py | 215 ----------------------- direct/src/p3d/ppackage.py | 23 +-- direct/src/plugin/p3dHost.cxx | 10 +- 6 files changed, 209 insertions(+), 382 deletions(-) delete mode 100755 direct/src/p3d/make_contents.py diff --git a/direct/src/p3d/FileSpec.py b/direct/src/p3d/FileSpec.py index b8c4f5f951..4c39a957a7 100644 --- a/direct/src/p3d/FileSpec.py +++ b/direct/src/p3d/FileSpec.py @@ -1,5 +1,5 @@ import os -from pandac.PandaModules import Filename, HashVal +from pandac.PandaModules import Filename, HashVal, VirtualFileSystem class FileSpec: """ This class represents a disk file whose hash and size @@ -7,7 +7,31 @@ class FileSpec: verify whether the file on disk matches the version demanded by the xml. """ - def __init__(self, xelement): + def __init__(self): + pass + + def fromFile(self, packageDir, filename): + """ Reads the file information from the indicated file. """ + vfs = VirtualFileSystem.getGlobalPtr() + + filename = Filename(filename) + pathname = Filename(packageDir, filename) + + self.filename = filename.cStr() + self.basename = filename.getBasename() + + st = os.stat(pathname.toOsSpecific()) + self.size = st.st_size + self.timestamp = st.st_mtime + + hv = HashVal() + hv.hashFile(pathname) + self.hash = hv.asHex() + + def loadXml(self, xelement): + """ Reads the file information from the indicated XML + element. """ + self.filename = xelement.Attribute('filename') self.basename = Filename(self.filename).getBasename() size = xelement.Attribute('size') @@ -23,6 +47,15 @@ class FileSpec: self.timestamp = 0 self.hash = xelement.Attribute('hash') + + def storeXml(self, xelement): + """ Adds the file information to the indicated XML + element. """ + + xelement.SetAttribute('filename', self.filename) + xelement.SetAttribute('size', str(self.size)) + xelement.SetAttribute('timestamp', str(self.timestamp)) + xelement.SetAttribute('hash', self.hash) def quickVerify(self, packageDir = None, pathname = None): """ Performs a quick test to ensure the file has not been diff --git a/direct/src/p3d/HostInfo.py b/direct/src/p3d/HostInfo.py index 3a99f48dd3..ba50c0d593 100644 --- a/direct/src/p3d/HostInfo.py +++ b/direct/src/p3d/HostInfo.py @@ -58,7 +58,8 @@ class HostInfo: platform = xpackage.Attribute('platform') version = xpackage.Attribute('version') package = self.__makePackage(name, platform, version) - package.descFile = FileSpec(xpackage) + package.descFile = FileSpec() + package.descFile.loadXml(xpackage) xpackage = xpackage.NextSiblingElement('package') @@ -69,7 +70,8 @@ class HostInfo: platform = ximport.Attribute('platform') version = ximport.Attribute('version') package = self.__makePackage(name, platform, version) - package.importDescFile = FileSpec(ximport) + package.importDescFile = FileSpec() + package.importDescFile.loadXml(ximport) ximport = ximport.NextSiblingElement('import') diff --git a/direct/src/p3d/Packager.py b/direct/src/p3d/Packager.py index dd84b6640d..fe1f1f0405 100644 --- a/direct/src/p3d/Packager.py +++ b/direct/src/p3d/Packager.py @@ -10,6 +10,7 @@ import marshal import new import string import types +from direct.p3d.FileSpec import FileSpec from direct.showbase import Loader from direct.showbase import AppRunnerGlobal from direct.showutil import FreezeTool @@ -130,6 +131,56 @@ class Packager: else: return self.glob.matches(filename.cStr()) + class PackageEntry: + """ This corresponds to an entry in the contents.xml file. """ + + def __init__(self): + pass + + def getKey(self): + """ Returns a tuple used for sorting the PackageEntry + objects uniquely per package. """ + return (self.packageName, self.platform, self.version) + + def fromFile(self, packageName, platform, version, solo, isImport, + installDir, descFilename): + self.packageName = packageName + self.platform = platform + self.version = version + self.solo = solo + self.isImport = isImport + + self.descFile = FileSpec() + self.descFile.fromFile(installDir, descFilename) + + def loadXml(self, xelement): + self.packageName = xelement.Attribute('name') + self.platform = xelement.Attribute('platform') + self.version = xelement.Attribute('version') + solo = xelement.Attribute('solo') + self.solo = int(solo or '0') + self.isImport = (xelement.Value() == 'import') + + self.descFile = FileSpec() + self.descFile.loadXml(xelement) + + def makeXml(self): + """ Returns a new TiXmlElement. """ + value = 'package' + if self.isImport: + value = 'import' + xelement = TiXmlElement(value) + xelement.SetAttribute('name', self.packageName) + if self.platform: + xelement.SetAttribute('platform', self.platform) + if self.version: + xelement.SetAttribute('version', self.version) + if self.solo: + xelement.SetAttribute('solo', '1') + + self.descFile.storeXml(xelement) + return xelement + class Package: def __init__(self, packageName, packager): self.packageName = packageName @@ -408,13 +459,26 @@ class Packager: self.writeDescFile() self.writeImportDescFile() + # Replace or add the entry in the contents. + a = Packager.PackageEntry() + a.fromFile(self.packageName, self.platform, self.version, + False, False, self.packager.installDir, + self.packageDesc) + + b = Packager.PackageEntry() + b.fromFile(self.packageName, self.platform, self.version, + False, True, self.packager.installDir, + self.packageImportDesc) + self.packager.contents[a.getKey()] = [a, b] + self.packager.contentsChanged = True + self.cleanup() def installSolo(self): """ Installs the package as a "solo", which means we - simply copy all files into the install directory. This is - primarily intended for the "coreapi" plugin, which is just - a single dll and a jpg file; but it can support other + simply copy the one file into the install directory. This + is primarily intended for the "coreapi" plugin, which is + just a single dll and a jpg file; but it can support other kinds of similar "solo" packages as well. """ packageDir = self.packageName @@ -430,22 +494,38 @@ class Packager: for origFile in origFiles: origFile.getFilename().unlink() - if not self.files: - # No files, never mind. - return - - Filename(installPath, '').makeDir() - + files = [] for file in self.files: if file.isExcluded(self): # Skip this file. continue - targetPath = Filename(installPath, file.newName) - targetPath.setBinary() - file.filename.setBinary() - if not file.filename.copyTo(targetPath): - print "Could not copy %s to %s" % ( - file.filename, targetPath) + files.append(file) + + if not files: + # No files, never mind. + return + + if len(files) != 1: + raise PackagerError, 'Multiple files in "solo" package %s' % (self.packageName) + + Filename(installPath, '').makeDir() + + file = files[0] + targetPath = Filename(installPath, file.newName) + targetPath.setBinary() + file.filename.setBinary() + if not file.filename.copyTo(targetPath): + print "Could not copy %s to %s" % ( + file.filename, targetPath) + + + # Replace or add the entry in the contents. + a = Packager.PackageEntry() + a.fromFile(self.packageName, self.platform, self.version, + True, False, self.packager.installDir, + Filename(packageDir, file.newName)) + self.packager.contents[a.getKey()] = [a] + self.packager.contentsChanged = True self.cleanup() @@ -1106,6 +1186,7 @@ class Packager: for moduleName, mdef in p2.moduleNames.items(): self.skipModules[moduleName] = mdef + # Packager constructor def __init__(self): # The following are config settings that the caller may adjust @@ -1268,6 +1349,10 @@ class Packager: # A table of all known packages by name. self.packages = {} + # A list of PackageEntry objects read from the contents.xml + # file. + self.contents = {} + def addWindowsSearchPath(self, searchPath, varname): """ Expands $varname, interpreting as a Windows-style search path, and adds its contents to the indicated DSearchPath. """ @@ -1303,133 +1388,13 @@ class Packager: if not PandaSystem.getPackageVersionString() or not PandaSystem.getPackageHostUrl(): raise PackagerError, 'This script must be run using a version of Panda3D that has been built\nfor distribution. Try using ppackage.p3d or packp3d.p3d instead.' - def __expandVariable(self, line, p): - """ Given that line[p] is a dollar sign beginning a variable - reference, advances p to the first dollar sign following the - reference, and looks up the variable referenced. + self.readContentsFile() - Returns (value, p) where value is the value of the named - variable, and p is the first character following the variable - reference. """ + def close(self): + """ Called after reading all of the package def files, this + performs any final cleanup appropriate. """ - p += 1 - if p >= len(line): - return '', p - - var = '' - if line[p] == '{': - # Curly braces exactly delimit the variable name. - p += 1 - while p < len(line) and line[p] != '}': - var += line[p] - p += 1 - else: - # Otherwise, a string of alphanumeric characters, - # including underscore, delimits the variable name. - var += line[p] - p += 1 - while p < len(line) and (line[p] in string.letters or line[p] in string.digits or line[p] == '_'): - var += line[p] - p += 1 - - return ExecutionEnvironment.getEnvironmentVariable(var), p - - - def __splitLine(self, line): - """ Separates the indicated line into words at whitespace. - Quotation marks and escape characters protect spaces. This is - designed to be similar to the algorithm employed by the Unix - shell. """ - - words = [] - - p = 0 - while p < len(line): - if line[p] == '#': - # A word that begins with a hash mark indicates an - # inline comment, and the end of the parsing. - break - - # Scan to the end of the word. - word = '' - while p < len(line) and line[p] not in string.whitespace: - if line[p] == '\\': - # Output an escaped character. - p += 1 - if p < len(line): - word += line[p] - p += 1 - - elif line[p] == '$': - # Output a variable reference. - expand, p = self.__expandVariable(line, p) - word += expand - - elif line[p] == '"': - # Output a double-quoted string. - p += 1 - while p < len(line) and line[p] != '"': - if line[p] == '\\': - # Output an escaped character. - p += 1 - if p < len(line): - word += line[p] - p += 1 - elif line[p] == '$': - # Output a variable reference. - expand, p = self.__expandVariable(line, p) - word += expand - else: - word += line[p] - p += 1 - - elif line[p] == "'": - # Output a single-quoted string. Escape - # characters and dollar signs within single quotes - # are not special. - p += 1 - while p < len(line) and line[p] != "'": - word += line[p] - p += 1 - - else: - # Output a single character. - word += line[p] - p += 1 - - words.append(word) - - # Scan to the beginning of the next word. - while p < len(line) and line[p] in string.whitespace: - p += 1 - - return words - - def __getNextLine(self): - """ Extracts the next line from self.inFile, and splits it - into words. Returns the list of words, or None at end of - file. """ - - line = self.inFile.readline() - self.lineNum += 1 - while line: - line = line.strip() - if not line: - # Skip the line, it was just a blank line - pass - - elif line[0] == '#': - # Eat python-style comments. - pass - - else: - return self.__splitLine(line) - - line = self.inFile.readline() - self.lineNum += 1 - - # End of file. - return None + self.writeContentsFile() def readPackageDef(self, packageDef): """ Reads the named .pdef file and constructs the packages @@ -1568,7 +1533,7 @@ class Packager: self.host = hostUrl # The descriptive name, if specified, is kept until the end, - # where it may be passed to make_contents by ppackage.py. + # where it may be written into the contents file. if descriptiveName: self.hostDescriptiveName = descriptiveName @@ -2188,6 +2153,59 @@ class Packager: explicit = False) + def readContentsFile(self): + """ Reads the contents.xml file at the beginning of + processing. """ + + self.contents = {} + self.contentsChanged = False + + contentsFilename = Filename(self.installDir, 'contents.xml') + doc = TiXmlDocument(contentsFilename.toOsSpecific()) + if not doc.LoadFile(): + # Couldn't read file. + return + + xcontents = doc.FirstChildElement('contents') + if xcontents: + if self.hostDescriptiveName is None: + self.hostDescriptiveName = xcontents.Attribute('descriptive_name') + + xelement = xcontents.FirstChildElement() + while xelement: + package = self.PackageEntry() + package.loadXml(xelement) + self.contents.setdefault(package.getKey(), []).append(package) + xelement = xelement.NextSiblingElement() + + def writeContentsFile(self): + """ Rewrites the contents.xml file at the end of + processing. """ + + if not self.contentsChanged: + # No need to rewrite. + return + + contentsFilename = Filename(self.installDir, 'contents.xml') + doc = TiXmlDocument(contentsFilename.toOsSpecific()) + decl = TiXmlDeclaration("1.0", "utf-8", "") + doc.InsertEndChild(decl) + + xcontents = TiXmlElement('contents') + if self.hostDescriptiveName: + xcontents.SetAttribute('descriptive_name', self.hostDescriptiveName) + + contents = self.contents.items() + contents.sort() + for key, entryList in contents: + for entry in entryList: + xelement = entry.makeXml() + xcontents.InsertEndChild(xelement) + + doc.InsertEndChild(xcontents) + doc.SaveFile() + + # The following class and function definitions represent a few sneaky # Python tricks to allow the pdef syntax to contain the pseudo-Python # code they do. These tricks bind the function and class definitions diff --git a/direct/src/p3d/make_contents.py b/direct/src/p3d/make_contents.py deleted file mode 100755 index 6386a108a5..0000000000 --- a/direct/src/p3d/make_contents.py +++ /dev/null @@ -1,215 +0,0 @@ -#! /usr/bin/env python - -""" -This command will build the contents.xml file at the top of a Panda3D -download hierarchy. This file lists all of the packages hosted here, -along with their current versions. - -This program runs on a local copy of the hosting directory hierarchy; -it must be a complete copy to generate a complete contents.xml file. - -make_contents.py [opts] - -Options: - - -i install_dir - The full path to a local directory that contains the - ready-to-be-published files, as populated by one or more - iterations of the ppackage script. It is the user's - responsibility to copy this directory structure to a server. - - -n "host descriptive name" - Specifies a descriptive name of the download server that will - host these contents. This name may be presented to the user when - managing installed packages. If this option is omitted, the name - is unchanged from the previous pass. - -""" - -import sys -import getopt -import os -import types - -try: - import hashlib -except ImportError: - # Legacy Python support - import md5 as hashlib - -class ArgumentError(AttributeError): - pass - -class FileSpec: - """ Represents a single file in the directory, and its associated - timestamp, size, and md5 hash. """ - - def __init__(self, filename, pathname): - self.filename = filename - self.pathname = pathname - - s = os.stat(pathname) - self.size = s.st_size - self.timestamp = int(s.st_mtime) - - m = hashlib.md5() - m.update(open(pathname, 'rb').read()) - self.hash = m.hexdigest() - - def getParams(self): - return 'filename="%s" size="%s" timestamp="%s" hash="%s"' % ( - self.filename, self.size, self.timestamp, self.hash) - -class ContentsMaker: - def __init__(self): - self.installDir = None - self.hostDescriptiveName = None - - def build(self): - if not self.installDir: - raise ArgumentError, "Stage directory not specified." - - self.packages = [] - self.scanDirectory() - - if not self.packages: - raise ArgumentError, "No packages found." - - contentsFileBasename = 'contents.xml' - contentsFilePathname = os.path.join(self.installDir, contentsFileBasename) - contentsLine = None - if self.hostDescriptiveName is not None: - if self.hostDescriptiveName: - contentsLine = '' % ( - self.quoteString(self.hostDescriptiveName)) - else: - contentsLine = self.readContentsLine(contentsFilePathname) - if not contentsLine: - contentsLine = '' - - # Now write the contents.xml file. - f = open(contentsFilePathname, 'w') - print >> f, '' - print >> f, '' - print >> f, contentsLine - for type, packageName, packagePlatform, packageVersion, file, solo in self.packages: - extra = '' - if solo: - extra += 'solo="1" ' - print >> f, ' <%s name="%s" platform="%s" version="%s" %s%s />' % ( - type, packageName, packagePlatform or '', packageVersion or '', extra, file.getParams()) - print >> f, '' - f.close() - - def readContentsLine(self, contentsFilePathname): - """ Reads the previous iteration of contents.xml to get the - previous top-level contents line, which contains the - hostDescriptiveName. """ - - try: - f = open(contentsFilePathname, 'r') - except IOError: - return None - - for line in f.readlines(): - if line.startswith('', '>') - return str - - def scanDirectory(self): - """ Walks through all the files in the stage directory and - looks for the package directory xml files. """ - - startDir = self.installDir - if startDir.endswith(os.sep): - startDir = startDir[:-1] - prefix = startDir + os.sep - for dirpath, dirnames, filenames in os.walk(startDir): - if dirpath == startDir: - localpath = '' - xml = '' - else: - assert dirpath.startswith(prefix) - localpath = dirpath[len(prefix):].replace(os.sep, '/') + '/' - xml = dirpath[len(prefix):].replace(os.sep, '_') + '.xml' - - solo = False - - # A special case: if a directory contains just one file, - # it's a "solo", not an xml package. - if len(filenames) == 1 and not filenames[0].endswith('.xml'): - xml = filenames[0] - solo = True - - if xml not in filenames: - continue - - if localpath.count('/') == 1: - packageName, junk = localpath.split('/') - packageVersion = None - packagePlatform = None - - elif localpath.count('/') == 2: - packageName, packageVersion, junk = localpath.split('/') - packagePlatform = None - - elif localpath.count('/') == 3: - packageName, packagePlatform, packageVersion, junk = localpath.split('/') - else: - continue - - file = FileSpec(localpath + xml, - os.path.join(self.installDir, localpath + xml)) - print file.filename - self.packages.append(('package', packageName, packagePlatform, packageVersion, file, solo)) - - if not solo: - # Look for an _import.xml file, too. - xml = xml[:-4] + '_import.xml' - try: - file = FileSpec(localpath + xml, - os.path.join(self.installDir, localpath + xml)) - except OSError: - file = None - if file: - print file.filename - self.packages.append(('import', packageName, packagePlatform, packageVersion, file, False)) - - -def makeContents(args): - opts, args = getopt.getopt(args, 'i:n:h') - - cm = ContentsMaker() - cm.installDir = '.' - for option, value in opts: - if option == '-i': - cm.installDir = value - - elif option == '-n': - cm.hostDescriptiveName = value - - elif option == '-h': - print __doc__ - sys.exit(1) - - cm.build() - - -if __name__ == '__main__': - try: - makeContents(sys.argv[1:]) - except ArgumentError, e: - print e.args[0] - sys.exit(1) diff --git a/direct/src/p3d/ppackage.py b/direct/src/p3d/ppackage.py index ef99785952..463a7a195b 100755 --- a/direct/src/p3d/ppackage.py +++ b/direct/src/p3d/ppackage.py @@ -87,7 +87,6 @@ import getopt import os from direct.p3d import Packager -from direct.p3d import make_contents from pandac.PandaModules import * def usage(code, msg = ''): @@ -112,9 +111,9 @@ for opt, arg in opts: elif opt == '-p': packager.platform = arg elif opt == '-u': - package.host = arg + packager.host = arg elif opt == '-n': - package.hostDescriptiveName = arg + packager.hostDescriptiveName = arg elif opt == '-h': usage(0) @@ -140,26 +139,10 @@ packager.installSearch.prependDirectory(packager.installDir) try: packager.setup() packages = packager.readPackageDef(packageDef) + packager.close() except Packager.PackagerError: # Just print the error message and exit gracefully. inst = sys.exc_info()[1] print inst.args[0] #raise sys.exit(1) - -# Look to see if we built any true packages, or if all of them were -# p3d files. -anyPackages = False -for package in packages: - if not package.p3dApplication: - anyPackages = True - break - -if anyPackages: - # If we built any true packages, then update the contents.xml at - # the root of the install directory. - cm = make_contents.ContentsMaker() - cm.installDir = packager.installDir.toOsSpecific() - cm.hostDescriptiveName = packager.hostDescriptiveName - cm.build() - diff --git a/direct/src/plugin/p3dHost.cxx b/direct/src/plugin/p3dHost.cxx index 4c3eddcf03..49bfa603be 100644 --- a/direct/src/plugin/p3dHost.cxx +++ b/direct/src/plugin/p3dHost.cxx @@ -163,7 +163,10 @@ get_package_desc_file(FileSpec &desc_file, // out const char *platform = xpackage->Attribute("platform"); const char *version = xpackage->Attribute("version"); const char *solo = xpackage->Attribute("solo"); - if (name != NULL && platform != NULL && version != NULL && + if (version == NULL) { + version = ""; + } + if (name != NULL && platform != NULL && package_name == name && inst_mgr->get_platform() == platform && package_version == version) { @@ -190,7 +193,10 @@ get_package_desc_file(FileSpec &desc_file, // out if (platform == NULL) { platform = ""; } - if (name != NULL && version != NULL && + if (version == NULL) { + version = ""; + } + if (name != NULL && package_name == name && *platform == '\0' && package_version == version) {