From da2f1cd9c985213622cb723ac122dc398ca6b712 Mon Sep 17 00:00:00 2001 From: David Rose Date: Wed, 19 Aug 2009 18:35:21 +0000 Subject: [PATCH] better frozen module handling --- direct/src/plugin/p3dInstance.cxx | 13 +- direct/src/plugin/p3dInstance.h | 1 + direct/src/plugin/p3dSession.cxx | 36 +++-- direct/src/showbase/Sources.pp | 16 +++ direct/src/showbase/VFSImporter.py | 99 ++++++++------ direct/src/showbase/extend_frozen.c | 204 ++++++++++++++++++++++++++++ direct/src/showutil/FreezeTool.py | 119 +++++++++++++--- direct/src/showutil/Packager.py | 66 ++++++--- direct/src/showutil/packp3d.py | 4 + direct/src/showutil/runp3d.py | 9 +- direct/src/stdpy/file.py | 6 +- 11 files changed, 474 insertions(+), 99 deletions(-) create mode 100644 direct/src/showbase/extend_frozen.c diff --git a/direct/src/plugin/p3dInstance.cxx b/direct/src/plugin/p3dInstance.cxx index dd9a2cf086..996724333f 100644 --- a/direct/src/plugin/p3dInstance.cxx +++ b/direct/src/plugin/p3dInstance.cxx @@ -67,13 +67,13 @@ P3DInstance(P3D_request_ready_func *func, P3DInstanceManager *inst_mgr = P3DInstanceManager::get_global_ptr(); _instance_id = inst_mgr->get_unique_id(); - - INIT_LOCK(_request_lock); - + _full_disk_access = false; _session = NULL; _panda3d = NULL; _splash_window = NULL; _instance_window_opened = false; + + INIT_LOCK(_request_lock); _requested_stop = false; #ifdef __APPLE__ @@ -844,11 +844,16 @@ scan_app_desc_file(TiXmlDocument *doc) { return; } - const char *log_basename = xpackage->Attribute("log"); + const char *log_basename = xpackage->Attribute("log_basename"); if (log_basename != NULL) { _log_basename = log_basename; } + int full_disk_access = 0; + if (xpackage->QueryIntAttribute("full_disk_access", &full_disk_access) == TIXML_SUCCESS) { + _full_disk_access = (full_disk_access != 0); + } + TiXmlElement *xrequires = xpackage->FirstChildElement("requires"); while (xrequires != NULL) { const char *name = xrequires->Attribute("name"); diff --git a/direct/src/plugin/p3dInstance.h b/direct/src/plugin/p3dInstance.h index 77a3580dc4..603f44e1b7 100644 --- a/direct/src/plugin/p3dInstance.h +++ b/direct/src/plugin/p3dInstance.h @@ -155,6 +155,7 @@ private: string _session_key; string _python_version; string _log_basename; + bool _full_disk_access; // Not ref-counted: session is the parent. P3DSession *_session; diff --git a/direct/src/plugin/p3dSession.cxx b/direct/src/plugin/p3dSession.cxx index bddecfc64e..56fca9ddda 100644 --- a/direct/src/plugin/p3dSession.cxx +++ b/direct/src/plugin/p3dSession.cxx @@ -652,7 +652,16 @@ start_p3dpython(P3DInstance *inst) { _python_root_dir = inst->_panda3d->get_package_dir(); - mkdir_complete(_start_dir, nout); + // We'll be changing the directory to the standard start directory + // only if we don't have full disk access set for the instance. If + // we do have this setting, we'll keep the current directory + // instead. + bool change_dir = !inst->_full_disk_access; + string start_dir; + if (change_dir) { + start_dir = _start_dir; + mkdir_complete(start_dir, nout); + } // Build up a search path that includes all of the required packages // that have already been installed. @@ -753,8 +762,8 @@ start_p3dpython(P3DInstance *inst) { string log_basename = inst->_log_basename; // But we also let it be overridden by the tokens. - if (inst->get_fparams().has_token("log")) { - log_basename = inst->get_fparams().lookup_token("log"); + if (inst->get_fparams().has_token("log_basename")) { + log_basename = inst->get_fparams().lookup_token("log_basename"); } // However, it is always written into the temp directory only; the @@ -790,12 +799,12 @@ start_p3dpython(P3DInstance *inst) { nout << "Attempting to start python from " << p3dpython << "\n"; #ifdef _WIN32 _p3dpython_handle = win_create_process - (p3dpython, _start_dir, env, _output_filename, + (p3dpython, start_dir, env, _output_filename, _pipe_read, _pipe_write); bool started_p3dpython = (_p3dpython_handle != INVALID_HANDLE_VALUE); #else _p3dpython_pid = posix_create_process - (p3dpython, _start_dir, env, _output_filename, + (p3dpython, start_dir, env, _output_filename, _pipe_read, _pipe_write); bool started_p3dpython = (_p3dpython_pid > 0); #endif @@ -1037,10 +1046,17 @@ win_create_process(const string &program, const string &start_dir, startup_info.wShowWindow = SW_HIDE; startup_info.dwFlags |= STARTF_USESHOWWINDOW; + // If the start directory is empty, meaning not to change the + // current directory, then pass NULL in to CreateProcess(). + const char *start_dir_cstr = NULL; + if (!start_dir.empty()) { + start_dir_cstr = start_dir.c_str(); + } + PROCESS_INFORMATION process_info; BOOL result = CreateProcess (program.c_str(), NULL, NULL, NULL, TRUE, 0, - (void *)env.c_str(), start_dir.c_str(), + (void *)env.c_str(), start_dir_cstr, &startup_info, &process_info); bool started_program = (result != 0); @@ -1129,9 +1145,11 @@ posix_create_process(const string &program, const string &start_dir, close(to_fd[1]); close(from_fd[0]); - if (chdir(start_dir.c_str()) < 0) { - nout << "Could not chdir to " << start_dir << "\n"; - _exit(1); + if (!start_dir.empty()) { + if (chdir(start_dir.c_str()) < 0) { + nout << "Could not chdir to " << start_dir << "\n"; + _exit(1); + } } // build up an array of char strings for the environment. diff --git a/direct/src/showbase/Sources.pp b/direct/src/showbase/Sources.pp index 271ad22b63..0975f58c67 100644 --- a/direct/src/showbase/Sources.pp +++ b/direct/src/showbase/Sources.pp @@ -21,6 +21,22 @@ #define IGATESCAN all #end lib_target +// Define a Python extension module for operating on frozen modules. +// This is a pure C module; it involves no Panda code or C++ code. +#begin lib_target + #define TARGET extend_frozen + #define LIB_PREFIX + #if $[OSX_PLATFORM] + #define LINK_AS_BUNDLE 1 + #define BUNDLE_EXT .so + #endif + #if $[WINDOWS_PLATFORM] + #define DYNAMIC_LIB_EXT .pyd + #endif + + #define SOURCES extend_frozen.c +#end lib_target + #if $[CTPROJS] #define INSTALL_SCRIPTS ppython #endif diff --git a/direct/src/showbase/VFSImporter.py b/direct/src/showbase/VFSImporter.py index f19f7e00ea..cf2b0ba61c 100644 --- a/direct/src/showbase/VFSImporter.py +++ b/direct/src/showbase/VFSImporter.py @@ -8,7 +8,7 @@ import imp import struct import __builtin__ -__all__ = ['register', 'reload_from', 'reload_packages'] +__all__ = ['register', 'freeze_new_modules'] vfs = VirtualFileSystem.getGlobalPtr() @@ -273,44 +273,65 @@ def register(): if not _registered: _registered = True sys.path_hooks.insert(0, VFSImporter) - -def reload_from(root_path, moduleName): - """ Reloads the named module from the indicated root directory, - merging it with the module already loaded, if any. This is - particularly useful for merging a VFS-mounted package with a - previously-frozen package. It allows you to release the initial - version of a package via the freeze mechanism, while still - allowing new additions to be added later via multifile. - - See also reload_packages(), which is a convenience function - wrapped around this one. """ - - path = root_path + '/' + '/'.join(moduleName.split('.')[:-1]) - importer = VFSImporter(path) - loader = importer.find_module(moduleName) - if loader: - loader.load_module(moduleName) -def reload_packages(multifile, root_path): - """ Walks the multifile and looks for Python packages that already - exist as frozen modules. For any such packages found, calls - reload_from() to merge them with the preloaded frozen package. """ +def freeze_new_modules(multifile, root_path): + """ Walks the multifile and looks for Python packages that are + children of frozen modules. These are converted to frozen + modules, since the Python runtime system only supports loading + frozen children of frozen modules. - for i in range(multifile.getNumSubfiles()): - filename = multifile.getSubfileName(i) - isInit = False - for ext in ['py'] + compiledExtensions: - if filename.endswith('/__init__.' + ext): - isInit = True - break - if not isInit: - continue + The multifile must be already mounted at root_path. """ - # Found a package. - moduleName = '.'.join(filename.split('/')[:-1]) - module = sys.modules.get(moduleName, None) - if module: - file = getattr(module, '__file__', None) - if file == '': - # It's a frozen module; replace it. - reload_from(root_path, moduleName) + # This module is defined by extend_frozen.c in + # direct/src/showbase. It's a special extension module that + # provides hooks into the array of frozen modules, which is + # otherwise accessible only to the C level. + import extend_frozen + + modules = [] + pyExtensions = ['py'] + compiledExtensions + for filename in multifile.getSubfileNames(): + filename = Filename(filename) + ext = filename.getExtension() + if ext in pyExtensions: + # A Python file. + moduleName = Filename(filename) + moduleName.setExtension('') + isPackage = False + if moduleName.getBasename() == '__init__': + # A package. + moduleName = moduleName.getDirname() + else: + moduleName = moduleName.cStr() + moduleName = '.'.join(moduleName.split('/')) + modules.append(moduleName) + + modules.sort() + + # Now look for any children of frozen modules; these children need + # to become frozen modules themselves. + existingFrozenModules = {} + newFrozen = [] + for moduleName in modules: + if extend_frozen.is_frozen_module(moduleName): + # It's a frozen module. All children require freezing also. + existingFrozenModules[moduleName] = True + else: + # It's not a frozen module, but maybe it needs to be. + if '.' in moduleName: + parentModuleName = moduleName.rsplit('.', 1)[0] + if parentModuleName in existingFrozenModules: + # Bad news. We have to freeze this one. + existingFrozenModules[moduleName] = True + + # Load up the module code. + path = root_path + '/' + '/'.join(moduleName.split('.')[:-1]) + importer = VFSImporter(path) + loader = importer.find_module(moduleName) + if loader: + code = loader.get_code(moduleName) + newFrozen.append((moduleName, marshal.dumps(code))) + + # Now pass our list of newly-frozen modules to the low level code. + if newFrozen: + extend_frozen.extend(newFrozen) diff --git a/direct/src/showbase/extend_frozen.c b/direct/src/showbase/extend_frozen.c new file mode 100644 index 0000000000..36ab3284aa --- /dev/null +++ b/direct/src/showbase/extend_frozen.c @@ -0,0 +1,204 @@ +#include + +#ifdef _WIN32 +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT +#endif + +/* + * Call this function to extend the frozen modules array with a new + * array of frozen modules, provided in a C-style array, at runtime. + * Returns the total number of frozen modules. + */ +static int +extend_frozen_modules(const struct _frozen *new_modules, int new_count) { + int orig_count; + struct _frozen *realloc_FrozenModules; + + /* First, count the number of frozen modules we had originally. */ + orig_count = 0; + while (PyImport_FrozenModules[orig_count].name != NULL) { + ++orig_count; + } + + if (new_count == 0) { + /* Trivial no-op. */ + return orig_count; + } + + /* Reallocate the PyImport_FrozenModules array bigger to make room + for the additional frozen modules. We just leak the original + array; it's too risky to try to free it. */ + realloc_FrozenModules = (struct _frozen *)malloc((orig_count + new_count + 1) * sizeof(struct _frozen)); + + /* The new frozen modules go at the front of the list. */ + memcpy(realloc_FrozenModules, new_modules, new_count * sizeof(struct _frozen)); + + /* Then the original set of frozen modules. */ + memcpy(realloc_FrozenModules + new_count, PyImport_FrozenModules, orig_count * sizeof(struct _frozen)); + + /* Finally, a single 0-valued entry marks the end of the array. */ + memset(realloc_FrozenModules + orig_count + new_count, 0, sizeof(struct _frozen)); + + /* Assign the new pointer. */ + PyImport_FrozenModules = realloc_FrozenModules; + + return orig_count + new_count; +} + +/* + * Call this function to extend the frozen modules array with a new + * list of frozen modules, provided in a Python-style list of (name, + * code) tuples, at runtime. This function is designed to be called + * from Python. + * + * Returns the total number of frozen modules. + */ +static PyObject * +py_extend_frozen_modules(PyObject *self, PyObject *args) { + PyObject *list; + int num_elements; + int i; + struct _frozen *new_modules; + + if (!PyArg_ParseTuple(args, "O", &list)) { + return NULL; + } + + if (!PySequence_Check(list)) { + Py_DECREF(list); + PyErr_SetString(PyExc_TypeError, "List required"); + return NULL; + } + + num_elements = PySequence_Size(list); + new_modules = (struct _frozen *)malloc(sizeof(struct _frozen) * num_elements); + + for (i = 0; i < num_elements; ++i) { + PyObject *tuple; + const char *name; + const char *code; + int size; + + tuple = PySequence_GetItem(list, i); + if (!PyArg_ParseTuple(tuple, "ss#", &name, &code, &size)) { + return NULL; + } + + /* We have to malloc new pointers for the name and code arrays. + These pointers will never be freed. */ + new_modules[i].name = strdup(name); + new_modules[i].code = (unsigned char *)malloc(size); + new_modules[i].size = size; + memcpy(new_modules[i].code, code, size); + + Py_DECREF(tuple); + } + + Py_DECREF(list); + + int total_count = extend_frozen_modules(new_modules, num_elements); + free(new_modules); + + return Py_BuildValue("i", total_count); +} + +/* + * Call this function to query whether the named module is already a + * frozen module or not. + */ +static PyObject * +py_is_frozen_module(PyObject *self, PyObject *args) { + const char *name; + int i; + + if (!PyArg_ParseTuple(args, "s", &name)) { + return NULL; + } + + i = 0; + while (PyImport_FrozenModules[i].name != NULL) { + if (strcmp(PyImport_FrozenModules[i].name, name) == 0) { + Py_INCREF(Py_True); + return Py_True; + } + ++i; + } + + Py_INCREF(Py_False); + return Py_False; +} + +/* + * This returns the tuple (code, isPackage), where code is the code + * string associated with the named frozen module, and isPackage is + * true if the module is a package, or false if it is a normal + * module). The return value is None if there is no such frozen + * module. You must use the marshal module to convert the code string + * to a code object. + */ +static PyObject * +py_get_frozen_module_code(PyObject *self, PyObject *args) { + const char *name; + int i; + + if (!PyArg_ParseTuple(args, "s", &name)) { + return NULL; + } + + i = 0; + while (PyImport_FrozenModules[i].name != NULL) { + if (strcmp(PyImport_FrozenModules[i].name, name) == 0) { + int is_package = (PyImport_FrozenModules[i].size < 0); + return Py_BuildValue("(s#i)", PyImport_FrozenModules[i].code, + abs(PyImport_FrozenModules[i].size), + is_package); + } + ++i; + } + + return Py_BuildValue(""); +} + +/* + * Call this function to return a list of the existing frozen module + * names. + */ +static PyObject * +py_get_frozen_module_names(PyObject *self, PyObject *args) { + int i; + PyObject *list; + + if (!PyArg_ParseTuple(args, "")) { + return NULL; + } + + list = PyList_New(0); + i = 0; + while (PyImport_FrozenModules[i].name != NULL) { + PyObject *name = PyString_FromString(PyImport_FrozenModules[i].name); + PyList_Append(list, name); + Py_DECREF(name); + ++i; + } + + return list; +} + +/* Initializes the Python module with our functions. */ +DLLEXPORT void initextend_frozen() { + static PyMethodDef extend_frozen_methods[] = { + { "extend", py_extend_frozen_modules, METH_VARARGS, + "Adds new frozen modules at runtime." }, + { "is_frozen_module", py_is_frozen_module, METH_VARARGS, + "Returns true if the named module is a frozen module." }, + { "get_frozen_module_code", py_get_frozen_module_code, METH_VARARGS, + "Returns the code string associated with the named module." }, + { "get_frozen_module_names", py_get_frozen_module_names, METH_VARARGS, + "Returns a list of frozen module names." }, + { NULL, NULL, 0, NULL } /* Sentinel */ + }; + + Py_InitModule("extend_frozen", extend_frozen_methods); +} diff --git a/direct/src/showutil/FreezeTool.py b/direct/src/showutil/FreezeTool.py index dd7fb9d6ca..c48d4a5c04 100644 --- a/direct/src/showutil/FreezeTool.py +++ b/direct/src/showutil/FreezeTool.py @@ -9,6 +9,12 @@ import imp import platform from distutils.sysconfig import PREFIX, get_python_inc, get_python_version +# Temporary (?) try..except to protect against unbuilt extend_frozen. +try: + import extend_frozen +except ImportError: + extend_frozen = None + import direct from pandac.PandaModules import * from pandac.extension_native_helpers import dll_suffix, dll_ext @@ -298,21 +304,49 @@ static PyMethodDef nullMethods[] = { {NULL, NULL} }; -%(dllexport)svoid init%(moduleName)s() { - int count; - struct _frozen *new_FrozenModules; +/* + * Call this function to extend the frozen modules array with a new + * array of frozen modules, provided in a C-style array, at runtime. + * Returns the total number of frozen modules. + */ +static int +extend_frozen_modules(const struct _frozen *new_modules, int new_count) { + int orig_count; + struct _frozen *realloc_FrozenModules; - count = 0; - while (PyImport_FrozenModules[count].name != NULL) { - ++count; + /* First, count the number of frozen modules we had originally. */ + orig_count = 0; + while (PyImport_FrozenModules[orig_count].name != NULL) { + ++orig_count; } - new_FrozenModules = (struct _frozen *)malloc((count + %(newcount)s + 1) * sizeof(struct _frozen)); - memcpy(new_FrozenModules, _PyImport_FrozenModules, %(newcount)s * sizeof(struct _frozen)); - memcpy(new_FrozenModules + %(newcount)s, PyImport_FrozenModules, count * sizeof(struct _frozen)); - memset(new_FrozenModules + count + %(newcount)s, 0, sizeof(struct _frozen)); - PyImport_FrozenModules = new_FrozenModules; + if (new_count == 0) { + /* Trivial no-op. */ + return orig_count; + } + /* Reallocate the PyImport_FrozenModules array bigger to make room + for the additional frozen modules. We just leak the original + array; it's too risky to try to free it. */ + realloc_FrozenModules = (struct _frozen *)malloc((orig_count + new_count + 1) * sizeof(struct _frozen)); + + /* The new frozen modules go at the front of the list. */ + memcpy(realloc_FrozenModules, new_modules, new_count * sizeof(struct _frozen)); + + /* Then the original set of frozen modules. */ + memcpy(realloc_FrozenModules + new_count, PyImport_FrozenModules, orig_count * sizeof(struct _frozen)); + + /* Finally, a single 0-valued entry marks the end of the array. */ + memset(realloc_FrozenModules + orig_count + new_count, 0, sizeof(struct _frozen)); + + /* Assign the new pointer. */ + PyImport_FrozenModules = realloc_FrozenModules; + + return orig_count + new_count; +} + +%(dllexport)svoid init%(moduleName)s() { + extend_frozen_modules(_PyImport_FrozenModules, %(newcount)s); Py_InitModule("%(moduleName)s", nullMethods); } """ @@ -525,6 +559,7 @@ class Freezer: try: module = __import__(moduleName) except: + print "couldn't import %s" % (moduleName) module = None if module != None: @@ -559,6 +594,7 @@ class Freezer: try: module = __import__(moduleName) except: + print "couldn't import %s" % (moduleName) module = None if module != None: @@ -733,7 +769,7 @@ class Freezer: self.modules[origName] = self.ModuleDef(origName, implicit = True) missing = [] - for origName in self.mf.any_missing(): + for origName in self.mf.any_missing_maybe()[0]: if origName in startupModules: continue if origName in self.previousModules: @@ -752,13 +788,15 @@ class Freezer: if prefix not in sourceTrees: # If it's in not one of our standard source trees, assume # it's some wacky system file we don't need. + print "ignoring missing %s" % (origName) continue missing.append(origName) - + if missing: error = "There are some missing modules: %r" % missing print error + print "previous = %s" % (self.previousModules,) raise StandardError, error def __loadModule(self, mdef): @@ -769,11 +807,23 @@ class Freezer: # disk. In this case, the moduleName may not be accurate # and useful, so load it as a file instead. + tempPath = None + if '.' not in mdef.moduleName: + # If we loaded a python file from the root, we need to + # temporarily add its directory to the module search + # path, so the modulefinder can find any sibling + # python files it imports as well. + tempPath = Filename(mdef.filename.getDirname()).toOsSpecific() + self.mf.path.append(tempPath) + pathname = mdef.filename.toOsSpecific() fp = open(pathname, modulefinder.READ_MODE) stuff = ("", "r", imp.PY_SOURCE) self.mf.load_module(mdef.moduleName, fp, pathname, stuff) + if tempPath: + del self.mf.path[-1] + else: # Otherwise, we can just import it normally. self.mf.import_hook(mdef.moduleName) @@ -1063,6 +1113,12 @@ class Freezer: filename = basename + self.sourceExtension + dllexport = '' + dllimport = '' + if self.platform == 'win32': + dllexport = '__declspec(dllexport) ' + dllimport = '__declspec(dllimport) ' + if compileToExe: code = self.frozenMainCode if self.platform == 'win32': @@ -1070,6 +1126,8 @@ class Freezer: initCode = self.mainInitCode % { 'frozenMainCode' : code, 'programName' : os.path.basename(basename), + 'dllexport' : dllexport, + 'dllimport' : dllimport, } if self.platform == 'win32': initCode += self.frozenExtensions @@ -1080,17 +1138,16 @@ class Freezer: compileFunc = self.compileExe else: - dllexport = '' if self.platform == 'win32': - dllexport = '__declspec(dllexport) ' target = basename + dllext + '.pyd' else: target = basename + '.so' initCode = dllInitCode % { - 'dllexport' : dllexport, 'moduleName' : os.path.basename(basename), 'newcount' : len(moduleList), + 'dllexport' : dllexport, + 'dllimport' : dllimport, } compileFunc = self.compileDll @@ -1215,21 +1272,43 @@ class PandaModuleFinder(modulefinder.ModuleFinder): try: return modulefinder.ModuleFinder.find_module(self, name, path, parent = parent) except ImportError: - # It wasn't found. Maybe it's one of ours. + # It wasn't found through the normal channels. Maybe it's + # one of ours, or maybe it's frozen? if path: # Only if we're not looking on a particular path, # though. raise - # This loop is roughly lifted from - # extension_native_helpers.Dtool_PreloadDLL(). + if extend_frozen and extend_frozen.is_frozen_module(name): + # It's a frozen module. + return (None, name, ('', '', imp.PY_FROZEN)) + + # Look for a dtool extension. This loop is roughly lifted + # from extension_native_helpers.Dtool_PreloadDLL(). filename = name + dll_suffix + dll_ext for dir in sys.path + [sys.prefix]: lib = os.path.join(dir, filename) if os.path.exists(lib): file = open(lib, 'rb') - return (file, lib, (dll_ext, 'rb', 3)) + return (file, lib, (dll_ext, 'rb', imp.C_EXTENSION)) message = "DLL loader cannot find %s." % (name) raise ImportError, message + def load_module(self, fqname, fp, pathname, (suffix, mode, type)): + if type == imp.PY_FROZEN: + # It's a frozen module. + co, isPackage = extend_frozen.get_frozen_module_code(pathname) + m = self.add_module(fqname) + m.__file__ = '' + if isPackage: + m.__path__ = pathname + co = marshal.loads(co) + if self.replace_paths: + co = self.replace_paths_in_code(co) + m.__code__ = co + self.scan_code(co, m) + self.msgout(2, "load_module ->", m) + return m + + return modulefinder.ModuleFinder.load_module(self, fqname, fp, pathname, (suffix, mode, type)) diff --git a/direct/src/showutil/Packager.py b/direct/src/showutil/Packager.py index 174f0daa68..5746901155 100644 --- a/direct/src/showutil/Packager.py +++ b/direct/src/showutil/Packager.py @@ -46,10 +46,20 @@ class Packager: if not self.newName: self.newName = self.filename.cStr() - packager = package.packager ext = Filename(self.newName).getExtension() + if ext == 'pz': + # Strip off a .pz extension; we can compress files + # within the Multifile without it. + filename = Filename(self.newName) + filename.setExtension('') + self.newName = filename.cStr() + ext = Filename(self.newName).getExtension() + if self.compress is None: + self.compress = True + + packager = package.packager if self.compress is None: - self.compress = (ext not in packager.uncompressibleExtensions) + self.compress = (ext not in packager.uncompressibleExtensions and ext not in packager.imageExtensions) if self.executable is None: self.executable = (ext in packager.executableExtensions) @@ -176,7 +186,7 @@ class Packager: # Add the explicit py files that were requested by the # pdef file. These get turned into Python modules. for file in self.files: - ext = file.filename.getExtension() + ext = Filename(file.newName).getExtension() if ext != 'py': continue @@ -252,13 +262,11 @@ class Packager: else: self.__addImplicitDependenciesPosix() - # Now add all the real, non-Python files. This will - # include the extension modules we just discovered above. - - # We walk through the list as we modify it. That's OK, - # because we may add new files that we want to process. + # Now add all the real, non-Python files (except model + # files). This will include the extension modules we just + # discovered above. for file in self.files: - ext = file.filename.getExtension() + ext = Filename(file.newName).getExtension() if ext == 'py': # Already handled, above. continue @@ -268,25 +276,38 @@ class Packager: continue if not self.dryRun: - if ext == 'pz': - # Strip off an implicit .pz extension. - filename = Filename(file.filename) - filename.setExtension('') - filename = Filename(filename.cStr()) - ext = filename.getExtension() + if ext == 'egg' or ext == 'bam': + # Skip model files this pass. + pass + else: + # Any other file. + self.addComponent(file) - filename = Filename(file.newName) - if filename.getExtension() == 'pz': - filename.setExtension('') - file.newName = filename.cStr() + # Finally, now add the model files. It's important to add + # these after we have added all of the texture files, so + # we can determine which textures need to be implicitly + # pulled in. + # We walk through the list as we modify it. That's OK, + # because we may add new files that we want to process. + for file in self.files: + ext = Filename(file.newName).getExtension() + if ext == 'py': + # Already handled, above. + continue + + if file.isExcluded(self): + # Skip this file. + continue + + if not self.dryRun: if ext == 'egg': self.addEggFile(file) elif ext == 'bam': self.addBamFile(file) else: - # Any other file. - self.addComponent(file) + # Handled above. + pass # Now that we've processed all of the component files, # (and set our platform if necessary), we can generate the @@ -838,7 +859,7 @@ class Packager: def addEggFile(self, file): # Precompile egg files to bam's. - np = self.packager.loader.loadModel(file.filename, okMissing = True) + np = self.packager.loader.loadModel(file.filename) if not np: raise StandardError, 'Could not read egg file %s' % (file.filename) @@ -938,6 +959,7 @@ class Packager: self.addFile(filename, newName = newName, explicit = False, compress = False) + return newName def addComponent(self, file): if file.platformSpecific: diff --git a/direct/src/showutil/packp3d.py b/direct/src/showutil/packp3d.py index 250a904e3b..222d6a0fe9 100755 --- a/direct/src/showutil/packp3d.py +++ b/direct/src/showutil/packp3d.py @@ -134,6 +134,9 @@ def main(appRunner): packp3d.p3d. """ print "args = %s" % (appRunner.argv,) + vfs = VirtualFileSystem.getGlobalPtr() + print "cwd = %s, %s" % (vfs.getCwd(), ExecutionEnvironment.getCwd()) + print "sys.path = %s" % (sys.path,) try: makePackedApp(appRunner.argv[1:]) except ArgumentError, e: @@ -142,6 +145,7 @@ def main(appRunner): sys.exit(0) if __name__ == '__main__': + print "sys.path = %s" % (sys.path,) try: makePackedApp(sys.argv[1:]) except ArgumentError, e: diff --git a/direct/src/showutil/runp3d.py b/direct/src/showutil/runp3d.py index ecfa88c1a3..400eea7259 100755 --- a/direct/src/showutil/runp3d.py +++ b/direct/src/showutil/runp3d.py @@ -191,9 +191,10 @@ class AppRunner(DirectObject): os.listdir = file.listdir os.walk = file.walk - # Make "/mf" our "current directory", for running the multifiles - # we plan to mount there. - vfs.chdir(MultifileRoot) + if not self.fullDiskAccess: + # Make "/mf" our "current directory", for running the multifiles + # we plan to mount there. + vfs.chdir(MultifileRoot) def startIfReady(self): if self.started: @@ -300,7 +301,7 @@ class AppRunner(DirectObject): # Mount the Multifile under /mf, by convention. vfs.mount(mf, MultifileRoot, vfs.MFReadOnly) - VFSImporter.reload_packages(mf, MultifileRoot) + VFSImporter.freeze_new_modules(mf, MultifileRoot) # Load any prc files in the root. We have to load them # explicitly, since the ConfigPageManager can't directly look diff --git a/direct/src/stdpy/file.py b/direct/src/stdpy/file.py index 8a7919c836..d70a5058aa 100644 --- a/direct/src/stdpy/file.py +++ b/direct/src/stdpy/file.py @@ -248,7 +248,11 @@ open = file def listdir(path): """ Implements os.listdir over vfs. """ files = [] - for file in _vfs.scanDirectory(path): + dirlist = _vfs.scanDirectory(path) + if dirlist is None: + message = 'No such file or directory: %s' % (path) + raise OSError, message + for file in dirlist: files.append(file.getFilename().getBasename()) return files