diff --git a/direct/src/showutil/FreezeTool.py b/direct/src/showutil/FreezeTool.py index 89016e274d..b47caa4825 100644 --- a/direct/src/showutil/FreezeTool.py +++ b/direct/src/showutil/FreezeTool.py @@ -8,7 +8,7 @@ import marshal import imp import platform from io import StringIO -from distutils.sysconfig import PREFIX, get_python_inc, get_python_version, get_config_var +import distutils.sysconfig as sysconf # Temporary (?) try..except to protect against unbuilt p3extend_frozen. try: @@ -29,9 +29,23 @@ isDebugBuild = (python.lower().endswith('_d')) # These are modules that Python always tries to import up-front. They # must be frozen in any main.exe. startupModules = [ - 'os', 'encodings.cp1252', - 'encodings.latin_1', 'encodings.utf_8', 'io', + 'encodings.cp1252', 'encodings.latin_1', 'encodings.utf_8', ] +if sys.version_info >= (3, 0): + startupModules += ['io', 'marshal', 'importlib.machinery', 'importlib.util'] + +# These are some special init functions for some built-in Python modules that +# deviate from the standard naming convention. A value of None means that a +# dummy entry should be written to the inittab. +builtinInitFuncs = { + 'builtins': None, + '__builtin__': None, + 'sys': None, + 'exceptions': None, + '_imp': 'PyInit_imp', + '_warnings': '_PyWarnings_Init', + 'marshal': 'PyMarshal_Init', +} # These are missing modules that we've reported already this session. reportedMissing = {} @@ -60,8 +74,8 @@ class CompilationEnvironment: # Paths to Python stuff. self.Python = None - self.PythonIPath = get_python_inc() - self.PythonVersion = get_config_var("LDVERSION") or get_python_version() + self.PythonIPath = sysconf.get_python_inc() + self.PythonVersion = sysconf.get_config_var("LDVERSION") or sysconf.get_python_version() # The VC directory of Microsoft Visual Studio (if relevant) self.MSVC = None @@ -85,7 +99,7 @@ class CompilationEnvironment: def determineStandardSetup(self): if self.platform.startswith('win'): - self.Python = PREFIX + self.Python = sysconf.PREFIX if ('VCINSTALLDIR' in os.environ): self.MSVC = os.environ['VCINSTALLDIR'] @@ -122,13 +136,15 @@ class CompilationEnvironment: # If it is run by makepanda, it handles the MSVC and PlatformSDK paths itself. if ('MAKEPANDA' in os.environ): - self.compileObj = 'cl /wd4996 /Fo%(basename)s.obj /nologo /c %(MD)s /Zi /O2 /Ob2 /EHsc /Zm300 /W3 /I"%(pythonIPath)s" %(filename)s' + self.compileObjExe = 'cl /wd4996 /Fo%(basename)s.obj /nologo /c %(MD)s /Zi /O2 /Ob2 /EHsc /Zm300 /W3 /I"%(pythonIPath)s" %(filename)s' + self.compileObjDll = self.compileObjExe self.linkExe = 'link /nologo /MAP:NUL /FIXED:NO /OPT:REF /STACK:4194304 /INCREMENTAL:NO /LIBPATH:"%(python)s\libs" /out:%(basename)s.exe %(basename)s.obj' self.linkDll = 'link /nologo /DLL /MAP:NUL /FIXED:NO /OPT:REF /INCREMENTAL:NO /LIBPATH:"%(python)s\libs" /out:%(basename)s%(dllext)s.pyd %(basename)s.obj' else: os.environ['PATH'] += ';' + self.MSVC + '\\bin' + self.suffix64 + ';' + self.MSVC + '\\Common7\\IDE;' + self.PSDK + '\\bin' - self.compileObj = 'cl /wd4996 /Fo%(basename)s.obj /nologo /c %(MD)s /Zi /O2 /Ob2 /EHsc /Zm300 /W3 /I"%(pythonIPath)s" /I"%(PSDK)s\include" /I"%(MSVC)s\include" %(filename)s' + self.compileObjExe = 'cl /wd4996 /Fo%(basename)s.obj /nologo /c %(MD)s /Zi /O2 /Ob2 /EHsc /Zm300 /W3 /I"%(pythonIPath)s" /I"%(PSDK)s\include" /I"%(MSVC)s\include" %(filename)s' + self.compileObjDll = self.compileObjExe self.linkExe = 'link /nologo /MAP:NUL /FIXED:NO /OPT:REF /STACK:4194304 /INCREMENTAL:NO /LIBPATH:"%(PSDK)s\lib" /LIBPATH:"%(MSVC)s\\lib%(suffix64)s" /LIBPATH:"%(python)s\libs" /out:%(basename)s.exe %(basename)s.obj' self.linkDll = 'link /nologo /DLL /MAP:NUL /FIXED:NO /OPT:REF /INCREMENTAL:NO /LIBPATH:"%(PSDK)s\lib" /LIBPATH:"%(MSVC)s\\lib%(suffix64)s" /LIBPATH:"%(python)s\libs" /out:%(basename)s%(dllext)s.pyd %(basename)s.obj' @@ -141,22 +157,26 @@ class CompilationEnvironment: self.arch = '-arch ppc' elif proc == 'amd64': self.arch = '-arch x86_64' - self.compileObj = "gcc -fPIC -c %(arch)s -o %(basename)s.o -O2 -I%(pythonIPath)s %(filename)s" + self.compileObjExe = "gcc -c %(arch)s -o %(basename)s.o -O2 -I%(pythonIPath)s %(filename)s" + self.compileObjDll = "gcc -fPIC -c %(arch)s -o %(basename)s.o -O2 -I%(pythonIPath)s %(filename)s" self.linkExe = "gcc %(arch)s -o %(basename)s %(basename)s.o -framework Python" self.linkDll = "gcc %(arch)s -undefined dynamic_lookup -bundle -o %(basename)s.so %(basename)s.o" else: # Unix - self.compileObj = "gcc -fPIC -c -o %(basename)s.o -O2 %(filename)s -I%(pythonIPath)s" - self.linkExe = "gcc -o %(basename)s %(basename)s.o -L/usr/local/lib -lpython%(pythonVersion)s" - self.linkDll = "gcc -shared -o %(basename)s.so %(basename)s.o -L/usr/local/lib -lpython%(pythonVersion)s" + lib_dir = sysconf.get_python_lib(plat_specific=1, standard_lib=1) + #python_a = os.path.join(lib_dir, "config", "libpython%(pythonVersion)s.a") + self.compileObjExe = "%(CC)s %(CFLAGS)s -c -o %(basename)s.o -pthread -O2 %(filename)s -I%(pythonIPath)s" + self.compileObjDll = "%(CC)s %(CFLAGS)s %(CCSHARED)s -c -o %(basename)s.o -O2 %(filename)s -I%(pythonIPath)s" + self.linkExe = "%(CC)s -o %(basename)s %(basename)s.o -L/usr/local/lib -lpython%(pythonVersion)s" + self.linkDll = "%(LDSHARED)s -o %(basename)s.so %(basename)s.o -L/usr/local/lib -lpython%(pythonVersion)s" if (os.path.isdir("/usr/PCBSD/local/lib")): self.linkExe += " -L/usr/PCBSD/local/lib" self.linkDll += " -L/usr/PCBSD/local/lib" - def compileExe(self, filename, basename): - compile = self.compileObj % { + def compileExe(self, filename, basename, extraLink=[]): + compile = self.compileObjExe % dict({ 'python' : self.Python, 'MSVC' : self.MSVC, 'PSDK' : self.PSDK, @@ -167,12 +187,12 @@ class CompilationEnvironment: 'arch' : self.arch, 'filename' : filename, 'basename' : basename, - } + }, **sysconf.get_config_vars()) sys.stderr.write(compile + '\n') if os.system(compile) != 0: raise Exception('failed to compile %s.' % basename) - link = self.linkExe % { + link = self.linkExe % dict({ 'python' : self.Python, 'MSVC' : self.MSVC, 'PSDK' : self.PSDK, @@ -182,13 +202,14 @@ class CompilationEnvironment: 'arch' : self.arch, 'filename' : filename, 'basename' : basename, - } + }, **sysconf.get_config_vars()) + link += ' ' + ' '.join(extraLink) sys.stderr.write(link + '\n') if os.system(link) != 0: raise Exception('failed to link %s.' % basename) - def compileDll(self, filename, basename): - compile = self.compileObj % { + def compileDll(self, filename, basename, extraLink=[]): + compile = self.compileObjDll % dict({ 'python' : self.Python, 'MSVC' : self.MSVC, 'PSDK' : self.PSDK, @@ -199,12 +220,12 @@ class CompilationEnvironment: 'arch' : self.arch, 'filename' : filename, 'basename' : basename, - } + }, **sysconf.get_config_vars()) sys.stderr.write(compile + '\n') if os.system(compile) != 0: raise Exception('failed to compile %s.' % basename) - link = self.linkDll % { + link = self.linkDll % dict({ 'python' : self.Python, 'MSVC' : self.MSVC, 'PSDK' : self.PSDK, @@ -215,7 +236,8 @@ class CompilationEnvironment: 'filename' : filename, 'basename' : basename, 'dllext' : self.dllext, - } + }, **sysconf.get_config_vars()) + link += ' ' + ' '.join(extraLink) sys.stderr.write(link + '\n') if os.system(link) != 0: raise Exception('failed to link %s.' % basename) @@ -264,6 +286,8 @@ Py_FrozenMain(int argc, char **argv) #endif Py_FrozenFlag = 1; /* Suppress errors from getpath.c */ + Py_NoSiteFlag = 1; + Py_NoUserSiteDirectory = 1; if ((p = Py_GETENV("PYTHONINSPECT")) && *p != '\\0') inspect = 1; @@ -463,10 +487,6 @@ main(int argc, char *argv[]) { # Our own glue code to start up a Python shared library. dllInitCode = """ -static PyMethodDef nullMethods[] = { - {NULL, NULL} -}; - /* * Call this function to extend the frozen modules array with a new * array of frozen modules, provided in a C-style array, at runtime. @@ -508,10 +528,29 @@ extend_frozen_modules(const struct _frozen *new_modules, int new_count) { return orig_count + new_count; } -%(dllexport)svoid init%(moduleName)s() { +#if PY_MAJOR_VERSION >= 3 +static PyModuleDef mdef = { + PyModuleDef_HEAD_INIT, + "%(moduleName)s", + "", + -1, + NULL, NULL, NULL, NULL, NULL +}; + +%(dllexport)sPyObject *PyInit_%(moduleName)s(void) { + extend_frozen_modules(_PyImport_FrozenModules, sizeof(_PyImport_FrozenModules) / sizeof(struct _frozen)); + return PyModule_Create(&mdef); +} +#else +static PyMethodDef nullMethods[] = { + {NULL, NULL} +}; + +%(dllexport)svoid init%(moduleName)s(void) { extend_frozen_modules(_PyImport_FrozenModules, sizeof(_PyImport_FrozenModules) / sizeof(struct _frozen)); Py_InitModule("%(moduleName)s", nullMethods); } +#endif """ programFile = """ @@ -526,8 +565,6 @@ struct _frozen _PyImport_FrozenModules[] = { %(moduleList)s {NULL, NULL, 0} }; - -%(initCode)s """ # Windows needs this bit. @@ -664,9 +701,14 @@ class Freezer: # addToMultifile(). It contains a list of all the extension # modules that were discovered, which have not been added to # the output. The list is a list of tuples of the form - # (moduleName, filename). + # (moduleName, filename). filename will be None for built-in + # modules. self.extras = [] + # Set this to true if extension modules should be linked in to + # the resulting executable. + self.linkExtensionModules = False + # End of public interface. These remaining members should not # be directly manipulated by callers. self.previousModules = {} @@ -676,6 +718,10 @@ class Freezer: self.previousModules = dict(previous.modules) self.modules = dict(previous.modules) + # Exclude doctest by default; it is not very useful in production + # builds. It can be explicitly included if desired. + self.modules['doctest'] = self.ModuleDef('doctest', exclude = True) + self.mf = None # Actually, make sure we know how to find all of the @@ -951,8 +997,8 @@ class Freezer: for mdef in includes: try: self.__loadModule(mdef) - except ImportError: - print("Unknown module: %s" % (mdef.moduleName)) + except ImportError as ex: + print("Unknown module: %s (%s)" % (mdef.moduleName, str(ex))) # Also attempt to import any implicit modules. If any of # these fail to import, we don't really care. @@ -1296,41 +1342,91 @@ class Freezer: if mdef.forbid: # Explicitly disallow importing this module. moduleList.append(self.makeForbiddenModuleListEntry(moduleName)) - else: - assert not mdef.exclude - # Allow importing this module. - module = self.mf.modules.get(origName, None) - code = getattr(module, "__code__", None) - if code: - code = marshal.dumps(code) + continue - mangledName = self.mangleName(moduleName) - moduleDefs.append(self.makeModuleDef(mangledName, code)) - moduleList.append(self.makeModuleListEntry(mangledName, code, moduleName, module)) + assert not mdef.exclude + # Allow importing this module. + module = self.mf.modules.get(origName, None) + code = getattr(module, "__code__", None) + if code: + code = marshal.dumps(code) - elif moduleName in startupModules: - # Forbid the loading of this startup module. - moduleList.append(self.makeForbiddenModuleListEntry(moduleName)) + mangledName = self.mangleName(moduleName) + moduleDefs.append(self.makeModuleDef(mangledName, code)) + moduleList.append(self.makeModuleListEntry(mangledName, code, moduleName, module)) + continue - else: - # This is a module with no associated Python - # code. It must be an extension module. Get the - # filename. - extensionFilename = getattr(module, '__file__', None) - if extensionFilename: - self.extras.append((moduleName, extensionFilename)) - else: - # It doesn't even have a filename; it must - # be a built-in module. No worries about - # this one, then. - pass + #if moduleName in startupModules: + # # Forbid the loading of this startup module. + # moduleList.append(self.makeForbiddenModuleListEntry(moduleName)) + # continue + + # This is a module with no associated Python code. It is either + # an extension module or a builtin module. Get the filename, if + # it is the former. + extensionFilename = getattr(module, '__file__', None) + + if extensionFilename or self.linkExtensionModules: + self.extras.append((moduleName, extensionFilename)) + + # If it is a submodule of a frozen module, Python will have + # trouble importing it as a builtin module. Synthesize a frozen + # module that loads it as builtin. + if '.' in moduleName and self.linkExtensionModules: + code = compile('import sys;del sys.modules["%s"];import imp;imp.init_builtin("%s")' % (moduleName, moduleName), moduleName, 'exec') + code = marshal.dumps(code) + mangledName = self.mangleName(moduleName) + moduleDefs.append(self.makeModuleDef(mangledName, code)) + moduleList.append(self.makeModuleListEntry(mangledName, code, moduleName, None)) + elif '.' in moduleName: + # Nothing we can do about this case except warn the user they + # are in for some trouble. + print('WARNING: Python cannot import extension modules under ' + 'frozen Python packages; %s will be inaccessible. ' + 'passing either -l to link in extension modules or use ' + '-x %s to exclude the entire package.' % (moduleName, moduleName.split('.')[0])) text = programFile % { 'moduleDefs': '\n'.join(moduleDefs), 'moduleList': '\n'.join(moduleList), - 'initCode': initCode } + if self.linkExtensionModules and self.extras: + # Python 3 case. + text += '#if PY_MAJOR_VERSION >= 3\n' + for module, fn in self.extras: + libName = module.split('.')[-1] + initFunc = builtinInitFuncs.get(module, 'PyInit_' + libName) + if initFunc: + text += 'extern PyObject *%s(void);\n' % (initFunc) + text += '\n' + text += 'struct _inittab _PyImport_Inittab[] = {\n' + for module, fn in self.extras: + libName = module.split('.')[-1] + initFunc = builtinInitFuncs.get(module, 'PyInit_' + libName) or 'NULL' + text += ' {"%s", %s},\n' % (module, initFunc) + text += ' {0, 0},\n' + text += '};\n\n' + + # Python 2 case. + text += '#else\n' + for module, fn in self.extras: + libName = module.split('.')[-1] + initFunc = builtinInitFuncs.get(module, 'init' + libName) + if initFunc: + text += 'extern void %s(void);\n' % (initFunc) + text += '\n' + text += 'struct _inittab _PyImport_Inittab[] = {\n' + for module, fn in self.extras: + libName = module.split('.')[-1] + initFunc = builtinInitFuncs.get(module, 'init' + libName) or 'NULL' + text += ' {"%s", %s},\n' % (module, initFunc) + text += ' {0, 0},\n' + text += '};\n' + text += '#endif\n\n' + + text += initCode + if filename is not None: file = open(filename, 'w') file.write(text) @@ -1397,8 +1493,14 @@ class Freezer: self.writeCode(filename, initCode=initCode) + extraLink = [] + if self.linkExtensionModules: + for mod, fn in self.extras: + if fn: + extraLink.append(fn) + try: - compileFunc(filename, basename) + compileFunc(filename, basename, extraLink=extraLink) finally: if not self.keepTemporaryFiles: if os.path.exists(filename): diff --git a/direct/src/showutil/pfreeze.py b/direct/src/showutil/pfreeze.py index 8676251026..fd150a1f19 100755 --- a/direct/src/showutil/pfreeze.py +++ b/direct/src/showutil/pfreeze.py @@ -15,7 +15,7 @@ imported directly or indirectly by the original startfile.py. Usage: - pfreeze.py [opts] startfile + pfreeze.py [opts] [startfile] Options: @@ -40,6 +40,11 @@ Options: of the __path__ variable, and thus must be actually imported to determine the true value of __path__. + -P path + Specifies an additional directory in which we should search for + Python modules. This is equivalent to setting the PYTHONPATH + environment variable. May be repeated. + -s Adds the standard set of modules that are necessary for embedding the Python interpreter. Implicitly set if an executable is @@ -55,7 +60,7 @@ from direct.showutil import FreezeTool def usage(code, msg = ''): if __doc__: sys.stderr.write(__doc__ + '\n') - sys.stderr.write(msg + '\n') + sys.stderr.write(str(msg) + '\n') sys.exit(code) # We're not protecting the next part under a __name__ == __main__ @@ -67,7 +72,7 @@ basename = None addStartupModules = False try: - opts, args = getopt.getopt(sys.argv[1:], 'o:i:x:p:sh') + opts, args = getopt.getopt(sys.argv[1:], 'o:i:x:p:P:slh') except getopt.error as msg: usage(1, msg) @@ -83,8 +88,12 @@ for opt, arg in opts: elif opt == '-p': for module in arg.split(','): freezer.handleCustomPath(module) + elif opt == '-P': + sys.path.append(arg) elif opt == '-s': addStartupModules = True + elif opt == '-l': + freezer.linkExtensionModules = True elif opt == '-h': usage(0) else: @@ -126,7 +135,7 @@ if args: elif outputType == 'exe': # We must have a main module when making an executable. - usage(0) + usage(1, 'A main file needs to be specified when creating an executable.') freezer.done(addStartupModules = addStartupModules)