More fixes to Python 3 rtdist, frozen p3dpython building, new open

direct/src/showbase/VFSImporter.py

@@ -25,12 +25,6 @@ sharedPackages = {}
 
 vfs = VirtualFileSystem.getGlobalPtr()
 
-# Possible file types.
-FTPythonSource = 0
-FTPythonCompiled = 1
-FTExtensionModule = 2
-FTFrozenModule = 3
-
 compiledExtensions = [ 'pyc', 'pyo' ]
 if not __debug__:
     # In optimized mode, we prefer loading .pyo files over .pyc files.
@@ -44,7 +38,10 @@ class VFSImporter:
     (among other places). """
 
     def __init__(self, path):
-        self.dir_path = Filename.fromOsSpecific(path)
+        if isinstance(path, Filename):
+            self.dir_path = Filename(path)
+        else:
+            self.dir_path = Filename.fromOsSpecific(path)
 
     def find_module(self, fullname, path = None):
         if path is None:
@@ -60,7 +57,8 @@ class VFSImporter:
         filename.setExtension('py')
         vfile = vfs.getFile(filename, True)
         if vfile:
-            return VFSLoader(dir_path, vfile, filename, FTPythonSource)
+            return VFSLoader(dir_path, vfile, filename,
+                             desc=('.py', 'U', imp.PY_SOURCE))
 
         # If there's no .py file, but there's a .pyc file, load that
         # anyway.
@@ -69,7 +67,8 @@ class VFSImporter:
             filename.setExtension(ext)
             vfile = vfs.getFile(filename, True)
             if vfile:
-                return VFSLoader(dir_path, vfile, filename, FTPythonCompiled)
+                return VFSLoader(dir_path, vfile, filename,
+                                 desc=('.'+ext, 'rb', imp.PY_COMPILED))
 
         # Look for a C/C++ extension module.
         for desc in imp.get_suffixes():
@@ -79,22 +78,21 @@ class VFSImporter:
             filename = Filename(path + desc[0])
             vfile = vfs.getFile(filename, True)
             if vfile:
-                return VFSLoader(dir_path, vfile, filename, FTExtensionModule,
+                return VFSLoader(dir_path, vfile, filename, desc=desc)
-                                 desc = desc)
 
         # Finally, consider a package, i.e. a directory containing
         # __init__.py.
         filename = Filename(path, '__init__.py')
         vfile = vfs.getFile(filename, True)
         if vfile:
-            return VFSLoader(dir_path, vfile, filename, FTPythonSource,
+            return VFSLoader(dir_path, vfile, filename, packagePath=path,
-                             packagePath = path)
+                             desc=('.py', 'U', imp.PY_SOURCE))
         for ext in compiledExtensions:
             filename = Filename(path, '__init__.' + ext)
             vfile = vfs.getFile(filename, True)
             if vfile:
-                return VFSLoader(dir_path, vfile, filename, FTPythonCompiled,
+                return VFSLoader(dir_path, vfile, filename, packagePath=path,
-                                 packagePath = path)
+                                 desc=('.'+ext, 'rb', imp.PY_COMPILED))
 
         #print >>sys.stderr, "not found."
         return None
@@ -103,22 +101,20 @@ class VFSLoader:
     """ The second part of VFSImporter, this is created for a
     particular .py file or directory. """
 
-    def __init__(self, dir_path, vfile, filename, fileType,
+    def __init__(self, dir_path, vfile, filename, desc, packagePath=None):
-                 desc = None, packagePath = None):
         self.dir_path = dir_path
         self.timestamp = None
         if vfile:
             self.timestamp = vfile.getTimestamp()
         self.filename = filename
-        self.fileType = fileType
         self.desc = desc
         self.packagePath = packagePath
 
     def load_module(self, fullname, loadingShared = False):
         #print >>sys.stderr, "load_module(%s), dir_path = %s, filename = %s" % (fullname, self.dir_path, self.filename)
-        if self.fileType == FTFrozenModule:
+        if self.desc[2] == imp.PY_FROZEN:
             return self._import_frozen_module(fullname)
-        if self.fileType == FTExtensionModule:
+        if self.desc[2] == imp.C_EXTENSION:
             return self._import_extension_module(fullname)
 
         # Check if this is a child of a shared package.
@@ -152,7 +148,7 @@ class VFSLoader:
         path = Filename(self.dir_path, Filename.fromOsSpecific(path))
         vfile = vfs.getFile(path)
         if not vfile:
-            raise IOError
+            raise IOError("Could not find '%s'" % (path))
         return vfile.readFile(True)
 
     def is_package(self, fullname):
@@ -171,8 +167,8 @@ class VFSLoader:
         """ Returns the Python source for this file, if it is
         available, or None if it is not.  May raise IOError. """
 
-        if self.fileType == FTPythonCompiled or \
+        if self.desc[2] == imp.PY_COMPILED or \
-           self.fileType == FTExtensionModule:
+           self.desc[2] == imp.C_EXTENSION:
             return None
 
         filename = Filename(self.filename)
@@ -180,7 +176,7 @@ class VFSLoader:
         filename.setText()
         vfile = vfs.getFile(filename)
         if not vfile:
-            raise IOError
+            raise IOError("Could not find '%s'" % (filename))
         return vfile.readFile(True)
 
     def _import_extension_module(self, fullname):
@@ -242,14 +238,14 @@ class VFSLoader:
         ValueError, SyntaxError, or a number of other errors generated
         by the low-level system. """
 
-        if self.fileType == FTPythonCompiled:
+        if self.desc[2] == imp.PY_COMPILED:
             # It's a pyc file; just read it directly.
             pycVfile = vfs.getFile(self.filename, False)
             if pycVfile:
                 return self._loadPyc(pycVfile, None)
             raise IOError('Could not read %s' % (self.filename))
 
-        elif self.fileType == FTExtensionModule:
+        elif self.desc[2] == imp.C_EXTENSION:
             return None
 
         # It's a .py file (or an __init__.py file; same thing).  Read

direct/src/showutil/FreezeTool.py

@@ -1386,6 +1386,10 @@ class PandaModuleFinder(modulefinder.ModuleFinder):
         modulefinder.ModuleFinder.__init__(self, *args, **kw)
 
     def find_module(self, name, path, *args, **kwargs):
+        if imp.is_frozen(name):
+            # Don't pick up modules that are frozen into p3dpython.
+            raise ImportError("'%s' is a frozen module" % (name))
+
         try:
             return modulefinder.ModuleFinder.find_module(self, name, path, *args, **kwargs)
         except ImportError:

makepanda/makepanda.py

@@ -5049,7 +5049,11 @@ if (RTDIST or RUNTIME):
     TargetAdd("libp3d_plugin_static.ilb", input='plugin_get_twirl_data.obj')
 
   if (PkgSkip("PYTHON")==0 and RTDIST):
-    TargetAdd('p3dpython_frozen.obj', opts=['DIR:direct/src/showbase', 'FREEZE_STARTUP'], input='VFSImporter.py')
+    # Freeze VFSImporter and its dependency modules into p3dpython.
+    # Mark panda3d.core as a dependency to make sure to build that first.
+    TargetAdd('p3dpython_frozen.obj', input='VFSImporter.py', opts=['DIR:direct/src/showbase', 'FREEZE_STARTUP'])
+    TargetAdd('p3dpython_frozen.obj', dep='panda3d/core.py')
+
     TargetAdd('p3dpython_p3dpython_composite1.obj', opts=OPTS, input='p3dpython_composite1.cxx')
     TargetAdd('p3dpython_p3dPythonMain.obj', opts=OPTS, input='p3dPythonMain.cxx')
     TargetAdd('p3dpython.exe', input='p3dpython_p3dpython_composite1.obj')

makepanda/makepandacore.py

@@ -2302,7 +2302,9 @@ def SetupBuildEnvironment(compiler):
     print("Target arch: %s" % GetTargetArch())
 
     # Set to English so we can safely parse the result of gcc commands.
-    os.environ["LC_ALL"] = "C"
+    # Setting it to UTF-8 is necessary for Python 3 modules to import
+    # correctly.
+    os.environ["LC_ALL"] = "en_US.UTF-8"
 
     if compiler == "MSVC":
         # Add the visual studio tools to PATH et al.
@@ -2842,11 +2844,11 @@ class Target:
 TARGET_LIST = []
 TARGET_TABLE = {}
 
-def TargetAdd(target, dummy=0, opts=0, input=0, dep=0, ipath=0, winrc=0):
+def TargetAdd(target, dummy=0, opts=[], input=[], dep=[], ipath=None, winrc=None):
     if (dummy != 0):
         exit("Syntax error in TargetAdd "+target)
-    if (ipath == 0): ipath = opts
+    if ipath is None: ipath = opts
-    if (ipath == 0): ipath = []
+    if not ipath: ipath = []
     if (type(input) == str): input = [input]
     if (type(dep) == str): dep = [dep]
 
@@ -2867,30 +2869,34 @@ def TargetAdd(target, dummy=0, opts=0, input=0, dep=0, ipath=0, winrc=0):
     else:
         t = TARGET_TABLE[full]
 
-    if opts != 0:
+    for x in opts:
-        for x in opts:
+        if x not in t.opts:
-            if (t.opts.count(x)==0):
+            t.opts.append(x)
-                t.opts.append(x)
 
     ipath = [OUTPUTDIR + "/tmp"] + GetListOption(ipath, "DIR:") + [OUTPUTDIR+"/include"]
-    if input != 0:
+    for x in input:
-        for x in input:
+        fullinput = FindLocation(x, ipath)
-            fullinput = FindLocation(x, ipath)
+        t.inputs.append(fullinput)
-            t.inputs.append(fullinput)
+        # Don't re-link a library or binary if just its dependency dlls have been altered.
-            # Don't re-link a library or binary if just its dependency dlls have been altered.
+        # This should work out fine in most cases, and often reduces recompilation time.
-            # This should work out fine in most cases, and often reduces recompilation time.
+        if (os.path.splitext(x)[-1] not in SUFFIX_DLL):
-            if (os.path.splitext(x)[-1] not in SUFFIX_DLL):
+            t.deps[fullinput] = 1
-                t.deps[fullinput] = 1
+            (base,suffix) = os.path.splitext(x)
-                (base,suffix) = os.path.splitext(x)
+            if (SUFFIX_INC.count(suffix)):
-                if (SUFFIX_INC.count(suffix)):
+                for d in CxxCalcDependencies(fullinput, ipath, []):
-                    for d in CxxCalcDependencies(fullinput, ipath, []):
+                    t.deps[d] = 1
-                        t.deps[d] = 1
-    if dep != 0:
-        for x in dep:
-            fulldep = FindLocation(x, ipath)
-            t.deps[fulldep] = 1
 
-    if winrc != 0 and GetTarget() == 'windows':
+        if x.endswith(".in"):
+            # Mark the _igate.cxx file as a dependency also.
+            outbase = os.path.basename(x)[:-3]
+            woutc = GetOutputDir()+"/tmp/"+outbase+"_igate.cxx"
+            t.deps[woutc] = 1
+
+    for x in dep:
+        fulldep = FindLocation(x, ipath)
+        t.deps[fulldep] = 1
+
+    if winrc and GetTarget() == 'windows':
         TargetAdd(target, input=WriteResourceFile(target.split("/")[-1].split(".")[0], **winrc))
 
     if target.endswith(".in"):

panda/src/express/streamWriter_ext.cxx

@@ -24,11 +24,10 @@
 ////////////////////////////////////////////////////////////////////
 void Extension<StreamWriter>::
 append_data(PyObject *data) {
-  cerr << "getting here: " << data << "\n";
   Py_buffer view;
   if (PyObject_GetBuffer(data, &view, PyBUF_CONTIG_RO) == -1) {
-    //PyErr_SetString(PyExc_TypeError,
+    PyErr_SetString(PyExc_TypeError,
-    //                "append_data() requires a contiguous buffer");
+                    "append_data() requires a contiguous buffer");
     return;
   }