dist: Support .* imports inside wheels

No longer try to import modules directly (we can do this if we really have to, but then we have to load it from the proper location) since we don't want to grab the system version of the package which may not be present or may be a different version.  Support discovering .* imports inside .whl files that are on sys.path.

direct/src/dist/FreezeTool.py

@@ -791,10 +791,6 @@ class Freezer:
         # default object will be created when it is needed.
         self.cenv = None
 
-        # This is the search path to use for Python modules.  Leave it
-        # to the default value of None to use sys.path.
-        self.path = path
-
         # The filename extension to append to the source file before
         # compiling.
         self.sourceExtension = '.c'
@@ -843,16 +839,14 @@ class Freezer:
         # builds.  It can be explicitly included if desired.
         self.modules['doctest'] = self.ModuleDef('doctest', exclude = True)
 
-        self.mf = None
-
         # Actually, make sure we know how to find all of the
         # already-imported modules.  (Some of them might do their own
         # special path mangling.)
         for moduleName, module in list(sys.modules.items()):
             if module and getattr(module, '__path__', None) is not None:
-                path = list(getattr(module, '__path__'))
+                modPath = list(getattr(module, '__path__'))
-                if path:
+                if modPath:
-                    modulefinder.AddPackagePath(moduleName, path[0])
+                    modulefinder.AddPackagePath(moduleName, modPath[0])
 
         # Module with non-obvious dependencies
         self.hiddenImports = defaultHiddenImports.copy()
@@ -861,14 +855,14 @@ class Freezer:
 
         # Suffix/extension for Python C extension modules
         if self.platform == PandaSystem.getPlatform():
-            self.moduleSuffixes = imp.get_suffixes()
+            suffixes = imp.get_suffixes()
 
             # Set extension for Python files to binary mode
-            for i, suffix in enumerate(self.moduleSuffixes):
+            for i, suffix in enumerate(suffixes):
                 if suffix[2] == imp.PY_SOURCE:
-                    self.moduleSuffixes[i] = (suffix[0], 'rb', imp.PY_SOURCE)
+                    suffixes[i] = (suffix[0], 'rb', imp.PY_SOURCE)
         else:
-            self.moduleSuffixes = [('.py', 'rb', 1), ('.pyc', 'rb', 2)]
+            suffixes = [('.py', 'rb', 1), ('.pyc', 'rb', 2)]
 
             abi_version = '{0}{1}'.format(*sys.version_info)
             abi_flags = ''
@@ -876,7 +870,7 @@ class Freezer:
                 abi_flags += 'm'
 
             if 'linux' in self.platform:
-                self.moduleSuffixes += [
+                suffixes += [
                     ('.cpython-{0}{1}-x86_64-linux-gnu.so'.format(abi_version, abi_flags), 'rb', 3),
                     ('.cpython-{0}{1}-i686-linux-gnu.so'.format(abi_version, abi_flags), 'rb', 3),
                     ('.abi{0}.so'.format(sys.version_info[0]), 'rb', 3),
@@ -884,24 +878,26 @@ class Freezer:
                 ]
             elif 'win' in self.platform:
                 # ABI flags are not appended on Windows.
-                self.moduleSuffixes += [
+                suffixes += [
                     ('.cp{0}-win_amd64.pyd'.format(abi_version), 'rb', 3),
                     ('.cp{0}-win32.pyd'.format(abi_version), 'rb', 3),
                     ('.pyd', 'rb', 3),
                 ]
             elif 'mac' in self.platform:
-                self.moduleSuffixes += [
+                suffixes += [
                     ('.cpython-{0}{1}-darwin.so'.format(abi_version, abi_flags), 'rb', 3),
                     ('.abi{0}.so'.format(sys.version_info[0]), 'rb', 3),
                     ('.so', 'rb', 3),
                 ]
             else: # FreeBSD et al.
-                self.moduleSuffixes += [
+                suffixes += [
                     ('.cpython-{0}{1}.so'.format(abi_version, abi_flags), 'rb', 3),
                     ('.abi{0}.so'.format(sys.version_info[0]), 'rb', 3),
                     ('.so', 'rb', 3),
                 ]
 
+        self.mf = PandaModuleFinder(excludes=['doctest'], suffixes=suffixes, path=path)
+
     def excludeFrom(self, freezer):
         """ Excludes all modules that have already been processed by
         the indicated FreezeTool.  This is equivalent to passing the
@@ -921,8 +917,6 @@ class Freezer:
         allowChildren is true, the children of the indicated module
         may still be included."""
 
-        assert self.mf is None
-
         self.modules[moduleName] = self.ModuleDef(
             moduleName, exclude = True,
             forbid = forbid, allowChildren = allowChildren,
@@ -947,24 +941,6 @@ class Freezer:
         files can be found.  If the module is a .py file and not a
         directory, returns None. """
 
-        # First, try to import the module directly.  That's the most
-        # reliable answer, if it works.
-        try:
-            module = __import__(moduleName)
-        except:
-            print("couldn't import %s" % (moduleName))
-            module = None
-
-        if module is not None:
-            for symbol in moduleName.split('.')[1:]:
-                module = getattr(module, symbol)
-            if hasattr(module, '__path__'):
-                return module.__path__
-
-        # If it didn't work--maybe the module is unimportable because
-        # it makes certain assumptions about the builtins, or
-        # whatever--then just look for file on disk.  That's usually
-        # good enough.
         path = None
         baseName = moduleName
         if '.' in baseName:
@@ -974,34 +950,20 @@ class Freezer:
                 return None
 
         try:
-            file, pathname, description = imp.find_module(baseName, path)
+            file, pathname, description = self.mf.find_module(baseName, path)
         except ImportError:
             return None
 
-        if not os.path.isdir(pathname):
+        if not self.mf._dir_exists(pathname):
             return None
+
         return [pathname]
 
     def getModuleStar(self, moduleName):
         """ Looks for the indicated directory module and returns the
         __all__ member: the list of symbols within the module. """
 
-        # First, try to import the module directly.  That's the most
+        # Open the directory and scan for *.py files.
-        # reliable answer, if it works.
-        try:
-            module = __import__(moduleName)
-        except:
-            print("couldn't import %s" % (moduleName))
-            module = None
-
-        if module is not None:
-            for symbol in moduleName.split('.')[1:]:
-                module = getattr(module, symbol)
-            if hasattr(module, '__all__'):
-                return module.__all__
-
-        # If it didn't work, just open the directory and scan for *.py
-        # files.
         path = None
         baseName = moduleName
         if '.' in baseName:
@@ -1011,16 +973,16 @@ class Freezer:
                 return None
 
         try:
-            file, pathname, description = imp.find_module(baseName, path)
+            file, pathname, description = self.mf.find_module(baseName, path)
         except ImportError:
             return None
 
-        if not os.path.isdir(pathname):
+        if not self.mf._dir_exists(pathname):
             return None
 
         # Scan the directory, looking for .py files.
         modules = []
-        for basename in sorted(os.listdir(pathname)):
+        for basename in sorted(self.mf._listdir(pathname)):
             if basename.endswith('.py') and basename != '__init__.py':
                 modules.append(basename[:-3])
 
@@ -1054,8 +1016,8 @@ class Freezer:
             modulePath = self.getModulePath(topName)
             if modulePath:
                 for dirname in modulePath:
-                    for basename in sorted(os.listdir(dirname)):
+                    for basename in sorted(self.mf._listdir(dirname)):
-                        if os.path.exists(os.path.join(dirname, basename, '__init__.py')):
+                        if self.mf._file_exists(os.path.join(dirname, basename, '__init__.py')):
                             parentName = '%s.%s' % (topName, basename)
                             newParentName = '%s.%s' % (newTopName, basename)
                             if self.getModulePath(parentName):
@@ -1100,8 +1062,6 @@ class Freezer:
         directories within a particular directory.
         """
 
-        assert self.mf is None
-
         if not newName:
             newName = moduleName
 
@@ -1122,8 +1082,6 @@ class Freezer:
         to done(), you may not add any more modules until you call
         reset(). """
 
-        assert self.mf is None
-
         # If we are building an exe, we also need to implicitly
         # bring in Python's startup modules.
         if addStartupModules:
@@ -1165,7 +1123,9 @@ class Freezer:
             else:
                 includes.append(mdef)
 
-        self.mf = PandaModuleFinder(excludes=list(excludeDict.keys()), suffixes=self.moduleSuffixes, path=self.path)
+        # Add the excludes to the ModuleFinder.
+        for exclude in excludeDict:
+            self.mf.excludes.append(exclude)
 
         # Attempt to import the explicit modules into the modulefinder.
 
@@ -2428,6 +2388,17 @@ class PandaModuleFinder(modulefinder.ModuleFinder):
 
         return None
 
+    def _file_exists(self, path):
+        if os.path.exists(path):
+            return os.path.isfile(path)
+
+        fh = self._open_file(path, 'rb')
+        if fh:
+            fh.close()
+            return True
+
+        return False
+
     def _dir_exists(self, path):
         """Returns True if the given directory exists, either on disk or inside
         a wheel."""
@@ -2466,6 +2437,43 @@ class PandaModuleFinder(modulefinder.ModuleFinder):
 
         return False
 
+    def _listdir(self, path):
+        """Lists files in the given directory if it exists."""
+
+        if os.path.isdir(path):
+            return os.listdir(path)
+
+        # Is there a zip file along the path?
+        dir, dirname = os.path.split(path.rstrip(os.path.sep + '/'))
+        fn = dirname
+        while dirname:
+            if os.path.isfile(dir):
+                # Okay, this is actually a file.  Is it a zip file?
+                if dir in self._zip_files:
+                    # Yes, and we've previously opened this.
+                    zip = self._zip_files[dir]
+                elif zipfile.is_zipfile(dir):
+                    zip = zipfile.ZipFile(dir)
+                    self._zip_files[dir] = zip
+                else:
+                    # It's not a directory or zip file.
+                    return []
+
+                # List files whose path start with our directory name.
+                prefix = fn.replace(os.path.sep, '/') + '/'
+                result = []
+                for name in zip.namelist():
+                    if name.startswith(prefix) and '/' not in name[len(prefix):]:
+                        result.append(name[len(prefix):])
+
+                return result
+
+            # Look at the parent directory.
+            dir, dirname = os.path.split(dir)
+            fn = os.path.join(dirname, fn)
+
+        return []
+
     def load_module(self, fqname, fp, pathname, file_info):
         """Copied from ModuleFinder.load_module with fixes to handle sending bytes
         to compile() for PY_SOURCE types. Sending bytes to compile allows it to
@@ -2712,7 +2720,7 @@ class PandaModuleFinder(modulefinder.ModuleFinder):
         modules = {}
         for dir in m.__path__:
             try:
-                names = os.listdir(dir)
+                names = self._listdir(dir)
             except OSError:
                 self.msg(2, "can't list directory", dir)
                 continue

direct/src/dist/commands.py

@@ -1023,7 +1023,7 @@ class build_apps(setuptools.Command):
 
             freezer_extras.update(freezer.extras)
             freezer_modules.update(freezer.getAllModuleNames())
-            for suffix in freezer.moduleSuffixes:
+            for suffix in freezer.mf.suffixes:
                 if suffix[2] == imp.C_EXTENSION:
                     ext_suffixes.add(suffix[0])