From 532bee3b9ef50c5080797f9454fa86767d9ad361 Mon Sep 17 00:00:00 2001 From: rdb Date: Wed, 23 Sep 2015 11:57:55 +0200 Subject: [PATCH] More fixes to Python 3 rtdist, frozen p3dpython building, new open --- direct/src/showbase/VFSImporter.py | 48 ++++++++++------------ direct/src/showutil/FreezeTool.py | 4 ++ makepanda/makepanda.py | 6 ++- makepanda/makepandacore.py | 56 ++++++++++++++------------ panda/src/express/streamWriter_ext.cxx | 5 +-- 5 files changed, 64 insertions(+), 55 deletions(-) diff --git a/direct/src/showbase/VFSImporter.py b/direct/src/showbase/VFSImporter.py index e7be781c45..e092775e6e 100644 --- a/direct/src/showbase/VFSImporter.py +++ b/direct/src/showbase/VFSImporter.py @@ -25,12 +25,6 @@ sharedPackages = {} vfs = VirtualFileSystem.getGlobalPtr() -# Possible file types. -FTPythonSource = 0 -FTPythonCompiled = 1 -FTExtensionModule = 2 -FTFrozenModule = 3 - compiledExtensions = [ 'pyc', 'pyo' ] if not __debug__: # In optimized mode, we prefer loading .pyo files over .pyc files. @@ -44,7 +38,10 @@ class VFSImporter: (among other places). """ def __init__(self, path): - self.dir_path = Filename.fromOsSpecific(path) + if isinstance(path, Filename): + self.dir_path = Filename(path) + else: + self.dir_path = Filename.fromOsSpecific(path) def find_module(self, fullname, path = None): if path is None: @@ -60,7 +57,8 @@ class VFSImporter: filename.setExtension('py') vfile = vfs.getFile(filename, True) if vfile: - return VFSLoader(dir_path, vfile, filename, FTPythonSource) + return VFSLoader(dir_path, vfile, filename, + desc=('.py', 'U', imp.PY_SOURCE)) # If there's no .py file, but there's a .pyc file, load that # anyway. @@ -69,7 +67,8 @@ class VFSImporter: filename.setExtension(ext) vfile = vfs.getFile(filename, True) if vfile: - return VFSLoader(dir_path, vfile, filename, FTPythonCompiled) + return VFSLoader(dir_path, vfile, filename, + desc=('.'+ext, 'rb', imp.PY_COMPILED)) # Look for a C/C++ extension module. for desc in imp.get_suffixes(): @@ -79,22 +78,21 @@ class VFSImporter: filename = Filename(path + desc[0]) vfile = vfs.getFile(filename, True) if vfile: - return VFSLoader(dir_path, vfile, filename, FTExtensionModule, - desc = desc) + return VFSLoader(dir_path, vfile, filename, desc=desc) # Finally, consider a package, i.e. a directory containing # __init__.py. filename = Filename(path, '__init__.py') vfile = vfs.getFile(filename, True) if vfile: - return VFSLoader(dir_path, vfile, filename, FTPythonSource, - packagePath = path) + return VFSLoader(dir_path, vfile, filename, packagePath=path, + desc=('.py', 'U', imp.PY_SOURCE)) for ext in compiledExtensions: filename = Filename(path, '__init__.' + ext) vfile = vfs.getFile(filename, True) if vfile: - return VFSLoader(dir_path, vfile, filename, FTPythonCompiled, - packagePath = path) + return VFSLoader(dir_path, vfile, filename, packagePath=path, + desc=('.'+ext, 'rb', imp.PY_COMPILED)) #print >>sys.stderr, "not found." return None @@ -103,22 +101,20 @@ class VFSLoader: """ The second part of VFSImporter, this is created for a particular .py file or directory. """ - def __init__(self, dir_path, vfile, filename, fileType, - desc = None, packagePath = None): + def __init__(self, dir_path, vfile, filename, desc, packagePath=None): self.dir_path = dir_path self.timestamp = None if vfile: self.timestamp = vfile.getTimestamp() self.filename = filename - self.fileType = fileType self.desc = desc self.packagePath = packagePath def load_module(self, fullname, loadingShared = False): #print >>sys.stderr, "load_module(%s), dir_path = %s, filename = %s" % (fullname, self.dir_path, self.filename) - if self.fileType == FTFrozenModule: + if self.desc[2] == imp.PY_FROZEN: return self._import_frozen_module(fullname) - if self.fileType == FTExtensionModule: + if self.desc[2] == imp.C_EXTENSION: return self._import_extension_module(fullname) # Check if this is a child of a shared package. @@ -152,7 +148,7 @@ class VFSLoader: path = Filename(self.dir_path, Filename.fromOsSpecific(path)) vfile = vfs.getFile(path) if not vfile: - raise IOError + raise IOError("Could not find '%s'" % (path)) return vfile.readFile(True) def is_package(self, fullname): @@ -171,8 +167,8 @@ class VFSLoader: """ Returns the Python source for this file, if it is available, or None if it is not. May raise IOError. """ - if self.fileType == FTPythonCompiled or \ - self.fileType == FTExtensionModule: + if self.desc[2] == imp.PY_COMPILED or \ + self.desc[2] == imp.C_EXTENSION: return None filename = Filename(self.filename) @@ -180,7 +176,7 @@ class VFSLoader: filename.setText() vfile = vfs.getFile(filename) if not vfile: - raise IOError + raise IOError("Could not find '%s'" % (filename)) return vfile.readFile(True) def _import_extension_module(self, fullname): @@ -242,14 +238,14 @@ class VFSLoader: ValueError, SyntaxError, or a number of other errors generated by the low-level system. """ - if self.fileType == FTPythonCompiled: + if self.desc[2] == imp.PY_COMPILED: # It's a pyc file; just read it directly. pycVfile = vfs.getFile(self.filename, False) if pycVfile: return self._loadPyc(pycVfile, None) raise IOError('Could not read %s' % (self.filename)) - elif self.fileType == FTExtensionModule: + elif self.desc[2] == imp.C_EXTENSION: return None # It's a .py file (or an __init__.py file; same thing). Read diff --git a/direct/src/showutil/FreezeTool.py b/direct/src/showutil/FreezeTool.py index cd303697f2..4b186d1eeb 100644 --- a/direct/src/showutil/FreezeTool.py +++ b/direct/src/showutil/FreezeTool.py @@ -1386,6 +1386,10 @@ class PandaModuleFinder(modulefinder.ModuleFinder): modulefinder.ModuleFinder.__init__(self, *args, **kw) def find_module(self, name, path, *args, **kwargs): + if imp.is_frozen(name): + # Don't pick up modules that are frozen into p3dpython. + raise ImportError("'%s' is a frozen module" % (name)) + try: return modulefinder.ModuleFinder.find_module(self, name, path, *args, **kwargs) except ImportError: diff --git a/makepanda/makepanda.py b/makepanda/makepanda.py index 464f798ced..b536140238 100755 --- a/makepanda/makepanda.py +++ b/makepanda/makepanda.py @@ -5049,7 +5049,11 @@ if (RTDIST or RUNTIME): TargetAdd("libp3d_plugin_static.ilb", input='plugin_get_twirl_data.obj') if (PkgSkip("PYTHON")==0 and RTDIST): - TargetAdd('p3dpython_frozen.obj', opts=['DIR:direct/src/showbase', 'FREEZE_STARTUP'], input='VFSImporter.py') + # Freeze VFSImporter and its dependency modules into p3dpython. + # Mark panda3d.core as a dependency to make sure to build that first. + TargetAdd('p3dpython_frozen.obj', input='VFSImporter.py', opts=['DIR:direct/src/showbase', 'FREEZE_STARTUP']) + TargetAdd('p3dpython_frozen.obj', dep='panda3d/core.py') + TargetAdd('p3dpython_p3dpython_composite1.obj', opts=OPTS, input='p3dpython_composite1.cxx') TargetAdd('p3dpython_p3dPythonMain.obj', opts=OPTS, input='p3dPythonMain.cxx') TargetAdd('p3dpython.exe', input='p3dpython_p3dpython_composite1.obj') diff --git a/makepanda/makepandacore.py b/makepanda/makepandacore.py index e5913a1c84..c9639ab1c7 100644 --- a/makepanda/makepandacore.py +++ b/makepanda/makepandacore.py @@ -2302,7 +2302,9 @@ def SetupBuildEnvironment(compiler): print("Target arch: %s" % GetTargetArch()) # Set to English so we can safely parse the result of gcc commands. - os.environ["LC_ALL"] = "C" + # Setting it to UTF-8 is necessary for Python 3 modules to import + # correctly. + os.environ["LC_ALL"] = "en_US.UTF-8" if compiler == "MSVC": # Add the visual studio tools to PATH et al. @@ -2842,11 +2844,11 @@ class Target: TARGET_LIST = [] TARGET_TABLE = {} -def TargetAdd(target, dummy=0, opts=0, input=0, dep=0, ipath=0, winrc=0): +def TargetAdd(target, dummy=0, opts=[], input=[], dep=[], ipath=None, winrc=None): if (dummy != 0): exit("Syntax error in TargetAdd "+target) - if (ipath == 0): ipath = opts - if (ipath == 0): ipath = [] + if ipath is None: ipath = opts + if not ipath: ipath = [] if (type(input) == str): input = [input] if (type(dep) == str): dep = [dep] @@ -2867,30 +2869,34 @@ def TargetAdd(target, dummy=0, opts=0, input=0, dep=0, ipath=0, winrc=0): else: t = TARGET_TABLE[full] - if opts != 0: - for x in opts: - if (t.opts.count(x)==0): - t.opts.append(x) + for x in opts: + if x not in t.opts: + t.opts.append(x) ipath = [OUTPUTDIR + "/tmp"] + GetListOption(ipath, "DIR:") + [OUTPUTDIR+"/include"] - if input != 0: - for x in input: - fullinput = FindLocation(x, ipath) - t.inputs.append(fullinput) - # Don't re-link a library or binary if just its dependency dlls have been altered. - # This should work out fine in most cases, and often reduces recompilation time. - if (os.path.splitext(x)[-1] not in SUFFIX_DLL): - t.deps[fullinput] = 1 - (base,suffix) = os.path.splitext(x) - if (SUFFIX_INC.count(suffix)): - for d in CxxCalcDependencies(fullinput, ipath, []): - t.deps[d] = 1 - if dep != 0: - for x in dep: - fulldep = FindLocation(x, ipath) - t.deps[fulldep] = 1 + for x in input: + fullinput = FindLocation(x, ipath) + t.inputs.append(fullinput) + # Don't re-link a library or binary if just its dependency dlls have been altered. + # This should work out fine in most cases, and often reduces recompilation time. + if (os.path.splitext(x)[-1] not in SUFFIX_DLL): + t.deps[fullinput] = 1 + (base,suffix) = os.path.splitext(x) + if (SUFFIX_INC.count(suffix)): + for d in CxxCalcDependencies(fullinput, ipath, []): + t.deps[d] = 1 - if winrc != 0 and GetTarget() == 'windows': + if x.endswith(".in"): + # Mark the _igate.cxx file as a dependency also. + outbase = os.path.basename(x)[:-3] + woutc = GetOutputDir()+"/tmp/"+outbase+"_igate.cxx" + t.deps[woutc] = 1 + + for x in dep: + fulldep = FindLocation(x, ipath) + t.deps[fulldep] = 1 + + if winrc and GetTarget() == 'windows': TargetAdd(target, input=WriteResourceFile(target.split("/")[-1].split(".")[0], **winrc)) if target.endswith(".in"): diff --git a/panda/src/express/streamWriter_ext.cxx b/panda/src/express/streamWriter_ext.cxx index 552b2a7dbd..68d7d39bc0 100644 --- a/panda/src/express/streamWriter_ext.cxx +++ b/panda/src/express/streamWriter_ext.cxx @@ -24,11 +24,10 @@ //////////////////////////////////////////////////////////////////// void Extension:: append_data(PyObject *data) { - cerr << "getting here: " << data << "\n"; Py_buffer view; if (PyObject_GetBuffer(data, &view, PyBUF_CONTIG_RO) == -1) { - //PyErr_SetString(PyExc_TypeError, - // "append_data() requires a contiguous buffer"); + PyErr_SetString(PyExc_TypeError, + "append_data() requires a contiguous buffer"); return; }