dist: Add an option to retain docstrings in compiled Python code

Fixes #1341

direct/src/dist/FreezeTool.py

@@ -794,7 +794,7 @@ class Freezer:
             return 'ModuleDef(%s)' % (', '.join(args))
 
     def __init__(self, previous = None, debugLevel = 0,
-                 platform = None, path=None):
+                 platform = None, path=None, optimize=None):
         # Normally, we are freezing for our own platform.  Change this
         # if untrue.
         self.platform = platform or PandaSystem.getPlatform()
@@ -859,6 +859,11 @@ class Freezer:
 
         self.mf = None
 
+        if optimize is None or optimize < 0:
+            self.optimize = sys.flags.optimize
+        else:
+            self.optimize = optimize
+
         # Actually, make sure we know how to find all of the
         # already-imported modules.  (Some of them might do their own
         # special path mangling.)
@@ -1174,7 +1179,7 @@ class Freezer:
             else:
                 includes.append(mdef)
 
-        self.mf = PandaModuleFinder(excludes=list(excludeDict.keys()), suffixes=self.moduleSuffixes, path=self.path)
+        self.mf = PandaModuleFinder(excludes=list(excludeDict.keys()), suffixes=self.moduleSuffixes, path=self.path, optimize=self.optimize)
 
         # Attempt to import the explicit modules into the modulefinder.
 
@@ -1445,6 +1450,9 @@ class Freezer:
                 else:
                     filename += '.pyo'
                 if multifile.findSubfile(filename) < 0:
+                    if sys.version_info >= (3, 2):
+                        code = compile('', moduleName, 'exec', optimize=self.optimize)
+                    else:
                         code = compile('', moduleName, 'exec')
                     self.__addPyc(multifile, filename, code, compressionLevel)
 
@@ -1525,6 +1533,9 @@ class Freezer:
                 source = open(sourceFilename.toOsSpecific(), 'r').read()
                 if source and source[-1] != '\n':
                     source = source + '\n'
+                if sys.version_info >= (3, 2):
+                    code = compile(source, str(sourceFilename), 'exec', optimize=self.optimize)
+                else:
                     code = compile(source, str(sourceFilename), 'exec')
 
         self.__addPyc(multifile, filename, code, compressionLevel)
@@ -1604,6 +1615,9 @@ class Freezer:
             # trouble importing it as a builtin module.  Synthesize a frozen
             # module that loads it as builtin.
             if '.' in moduleName and self.linkExtensionModules:
+                if sys.version_info >= (3, 2):
+                    code = compile('import sys;del sys.modules["%s"];import imp;imp.init_builtin("%s")' % (moduleName, moduleName), moduleName, 'exec', optimize=self.optimize)
+                else:
                     code = compile('import sys;del sys.modules["%s"];import imp;imp.init_builtin("%s")' % (moduleName, moduleName), moduleName, 'exec')
                 code = marshal.dumps(code)
                 mangledName = self.mangleName(moduleName)
@@ -1881,7 +1895,7 @@ class Freezer:
                 else:
                     code = 'import sys;del sys.modules["%s"];import sys,os,imp;imp.load_dynamic("%s",os.path.join(os.path.dirname(sys.executable), "%s%s"))' % (moduleName, moduleName, moduleName, modext)
                 if sys.version_info >= (3, 2):
-                    code = compile(code, moduleName, 'exec', optimize=2)
+                    code = compile(code, moduleName, 'exec', optimize=self.optimize)
                 else:
                     code = compile(code, moduleName, 'exec')
                 code = marshal.dumps(code)
@@ -1969,6 +1983,8 @@ class Freezer:
                 flags |= 1
             if log_filename_strftime:
                 flags |= 2
+            if self.optimize < 2:
+                flags |= 4 # keep_docstrings
 
             # Compose the header we will be writing to the stub, to tell it
             # where to find the module data blob, as well as other variables.
@@ -2341,6 +2357,7 @@ class PandaModuleFinder(modulefinder.ModuleFinder):
         """
 
         self.suffixes = kw.pop('suffixes', imp.get_suffixes())
+        self.optimize = kw.pop('optimize', -1)
 
         modulefinder.ModuleFinder.__init__(self, *args, **kw)
 
@@ -2446,6 +2463,9 @@ class PandaModuleFinder(modulefinder.ModuleFinder):
 
         if type is _PKG_NAMESPACE_DIRECTORY:
             m = self.add_module(fqname)
+            if sys.version_info >= (3, 2):
+                m.__code__ = compile('', '', 'exec', optimize=self.optimize)
+            else:
                 m.__code__ = compile('', '', 'exec')
             m.__path__ = pathname
             return m
@@ -2458,6 +2478,9 @@ class PandaModuleFinder(modulefinder.ModuleFinder):
                 code = fp.read()
 
             code += b'\n' if isinstance(code, bytes) else '\n'
+            if sys.version_info >= (3, 2):
+                co = compile(code, pathname, 'exec', optimize=self.optimize)
+            else:
                 co = compile(code, pathname, 'exec')
         elif type == imp.PY_COMPILED:
             if sys.version_info >= (3, 7):

direct/src/dist/commands.py

@@ -252,6 +252,7 @@ class build_apps(setuptools.Command):
         self.log_append = False
         self.prefer_discrete_gpu = False
         self.requirements_path = os.path.join(os.getcwd(), 'requirements.txt')
+        self.strip_docstrings = True
         self.use_optimized_wheels = True
         self.optimized_wheel_index = ''
         self.pypi_extra_indexes = [
@@ -776,7 +777,9 @@ class build_apps(setuptools.Command):
                 # Legacy handling for Tcl data files
                 site_py += SITE_PY_TKINTER_ADDENDUM
 
-            freezer = FreezeTool.Freezer(platform=platform, path=path)
+            optimize = 2 if self.strip_docstrings else 1
+
+            freezer = FreezeTool.Freezer(platform=platform, path=path, optimize=optimize)
             freezer.addModule('__main__', filename=mainscript)
             freezer.addModule('site', filename='site.py', text=site_py)
             for incmod in self.include_modules.get(appname, []) + self.include_modules.get('*', []):

pandatool/src/deploy-stub/deploy-stub.c

@@ -40,6 +40,7 @@
 enum Flags {
   F_log_append = 1,
   F_log_filename_strftime = 2,
+  F_keep_docstrings = 4,
 };
 
 /* Define an exposed symbol where we store the offset to the module data. */
@@ -449,7 +450,11 @@ int Py_FrozenMain(int argc, char **argv)
     Py_NoUserSiteDirectory = 1;
 
 #if PY_VERSION_HEX >= 0x03020000
+    if (blobinfo.flags & F_keep_docstrings) {
+      Py_OptimizeFlag = 1;
+    } else {
       Py_OptimizeFlag = 2;
+    }
 #endif
 
 #ifndef NDEBUG