Add ability to produce .whl file

Based on original version by pennomi
Closes: #83

makepanda/makepanda.py

@@ -39,6 +39,7 @@ import sys
 
 COMPILER=0
 INSTALLER=0
+WHEEL=0
 GENMAN=0
 COMPRESSOR="zlib"
 THREADCOUNT=0
@@ -124,6 +125,7 @@ def usage(problem):
     print("  --verbose         (print out more information)")
     print("  --runtime         (build a runtime build instead of an SDK build)")
     print("  --installer       (build an installer)")
+    print("  --wheel           (build a pip-installable .whl)")
     print("  --optimize X      (optimization level can be 1,2,3,4)")
     print("  --version X       (set the panda version number)")
     print("  --lzma            (use lzma compression when building Windows installer)")
@@ -159,13 +161,13 @@ def usage(problem):
     os._exit(1)
 
 def parseopts(args):
-    global INSTALLER,RTDIST,RUNTIME,GENMAN,DISTRIBUTOR,VERSION
+    global INSTALLER,WHEEL,RTDIST,RUNTIME,GENMAN,DISTRIBUTOR,VERSION
     global COMPRESSOR,THREADCOUNT,OSXTARGET,OSX_ARCHS,HOST_URL
     global DEBVERSION,RPMRELEASE,GIT_COMMIT,P3DSUFFIX,RTDIST_VERSION
     global STRDXSDKVERSION, WINDOWS_SDK, MSVC_VERSION, BOOUSEINTELCOMPILER
     longopts = [
         "help","distributor=","verbose","runtime","osxtarget=",
-        "optimize=","everything","nothing","installer","rtdist","nocolor",
+        "optimize=","everything","nothing","installer","wheel","rtdist","nocolor",
         "version=","lzma","no-python","threads=","outputdir=","override=",
         "static","host=","debversion=","rpmrelease=","p3dsuffix=","rtdist-version=",
         "directx-sdk=", "windows-sdk=", "msvc-version=", "clean", "use-icl",
@@ -188,6 +190,7 @@ def parseopts(args):
             if (option=="--help"): raise Exception
             elif (option=="--optimize"): optimize=value
             elif (option=="--installer"): INSTALLER=1
+            elif (option=="--wheel"): WHEEL=1
             elif (option=="--verbose"): SetVerbose(True)
             elif (option=="--distributor"): DISTRIBUTOR=value
             elif (option=="--rtdist"): RTDIST=1
@@ -416,9 +419,18 @@ if (RUNTIME):
 if (INSTALLER and RTDIST):
     exit("Cannot build an installer for the rtdist build!")
 
+if (WHEEL and RUNTIME):
+    exit("Cannot build a wheel for the runtime build!")
+
+if (WHEEL and RTDIST):
+    exit("Cannot build a wheel for the rtdist build!")
+
 if (INSTALLER) and (PkgSkip("PYTHON")) and (not RUNTIME) and GetTarget() == 'windows':
     exit("Cannot build installer on Windows without python")
 
+if WHEEL and PkgSkip("PYTHON"):
+    exit("Cannot build wheel without Python")
+
 if (RTDIST) and (PkgSkip("WX") and PkgSkip("FLTK")):
     exit("Cannot build rtdist without wx or fltk")
 
@@ -7239,6 +7251,11 @@ try:
             MakeInstallerFreeBSD()
         else:
             exit("Do not know how to make an installer for this platform")
+
+    if WHEEL:
+        ProgressOutput(100.0, "Building wheel")
+        from makewheel import makewheel
+        makewheel(VERSION, GetOutputDir())
 finally:
     SaveDependencyCache()
 

makepanda/makewheel.py

@@ -0,0 +1,558 @@
+"""
+Generates a wheel (.whl) file from the output of makepanda.
+
+Since the wheel requires special linking, this will only work if compiled with
+the `--wheel` parameter.
+"""
+from __future__ import print_function, unicode_literals
+from distutils.util import get_platform as get_dist
+import json
+
+import sys
+import os
+from os.path import join
+import shutil
+import zipfile
+import hashlib
+import tempfile
+import subprocess
+from sysconfig import get_config_var
+from optparse import OptionParser
+from makepandacore import ColorText, LocateBinary, ParsePandaVersion, GetExtensionSuffix, SetVerbose, GetVerbose
+from base64 import urlsafe_b64encode
+
+
+def get_platform():
+    p = get_dist().replace('-', '_').replace('.', '_')
+    #if "linux" in p:
+    #    print(ColorText("red", "WARNING:") +
+    #          " Linux-specific wheel files are not supported."
+    #          " We will generate this wheel as a generic package instead.")
+    #    return "any"
+    return p
+
+
+def get_abi_tag():
+    if sys.version_info >= (3, 0):
+        soabi = get_config_var('SOABI')
+        if soabi and soabi.startswith('cpython-'):
+            return 'cp' + soabi.split('-')[1]
+        elif soabi:
+            return soabi.replace('.', '_').replace('-', '_')
+
+    soabi = 'cp%d%d' % (sys.version_info[:2])
+
+    debug_flag = get_config_var('Py_DEBUG')
+    if (debug_flag is None and hasattr(sys, 'gettotalrefcount')) or debug_flag:
+        soabi += 'd'
+
+    malloc_flag = get_config_var('WITH_PYMALLOC')
+    if malloc_flag is None or malloc_flag:
+        soabi += 'm'
+
+    if sys.version_info < (3, 3):
+        usize = get_config_var('Py_UNICODE_SIZE')
+        if (usize is None and sys.maxunicode == 0x10ffff) or usize == 4:
+            soabi += 'u'
+
+    return soabi
+
+
+def is_exe_file(path):
+    return os.path.isfile(path) and path.lower().endswith('.exe')
+
+
+def is_elf_file(path):
+    base = os.path.basename(path)
+    return os.path.isfile(path) and '.' not in base and \
+           open(path, 'rb').read(4) == b'\x7FELF'
+
+
+def is_mach_o_file(path):
+    base = os.path.basename(path)
+    return os.path.isfile(path) and '.' not in base and \
+           open(path, 'rb').read(4) == b'\xCA\xFE\xBA\xBE'
+
+
+if sys.platform in ('win32', 'cygwin'):
+    is_executable = is_exe_file
+elif sys.platform == 'darwin':
+    is_executable = is_mach_o_file
+else:
+    is_executable = is_elf_file
+
+
+# Other global parameters
+PY_VERSION = "cp{}{}".format(sys.version_info.major, sys.version_info.minor)
+ABI_TAG = get_abi_tag()
+PLATFORM_TAG = get_platform()
+EXCLUDE_EXT = [".pyc", ".pyo", ".N", ".prebuilt", ".xcf", ".plist", ".vcproj", ".sln"]
+
+# Plug-ins to install.
+PLUGIN_LIBS = ["pandagl", "pandagles", "pandagles2", "p3ptloader", "p3assimp", "p3ffmpeg", "p3openal_audio", "p3fmod_audio"]
+
+WHEEL_DATA = """Wheel-Version: 1.0
+Generator: makepanda
+Root-Is-Purelib: false
+Tag: {}-{}-{}
+"""
+
+METADATA = {
+    "license": "BSD",
+    "name": "Panda3D",
+    "metadata_version": "2.0",
+    "generator": "makepanda",
+    "summary": "Panda3D is a game engine, a framework for 3D rendering and "
+               "game development for Python and C++ programs.",
+    "extensions": {
+        "python.details": {
+            "project_urls": {
+                "Home": "https://www.panda3d.org/"
+            },
+            "document_names": {
+                "license": "LICENSE.txt"
+            },
+            "contacts": [
+                {
+                    "role": "author",
+                    "email": "etc-panda3d@lists.andrew.cmu.edu",
+                    "name": "Panda3D Team"
+                }
+            ]
+        }
+    },
+    "classifiers": [
+        "Development Status :: 5 - Production/Stable",
+        "Intended Audience :: Developers",
+        "Intended Audience :: End Users/Desktop",
+        "License :: OSI Approved :: BSD License",
+        "Operating System :: OS Independent",
+        "Programming Language :: C++",
+        "Programming Language :: Python",
+        "Topic :: Games/Entertainment",
+        "Topic :: Multimedia",
+        "Topic :: Multimedia :: Graphics",
+        "Topic :: Multimedia :: Graphics :: 3D Rendering"
+    ]
+}
+
+PANDA3D_TOOLS_INIT = """import os, sys
+import panda3d
+
+if sys.platform in ('win32', 'cygwin'):
+    path_var = 'PATH'
+elif sys.platform == 'darwin':
+    path_var = 'DYLD_LIBRARY_PATH'
+else:
+    path_var = 'LD_LIBRARY_PATH'
+
+dir = os.path.dirname(panda3d.__file__)
+del panda3d
+if not os.environ.get(path_var):
+    os.environ[path_var] = dir
+else:
+    os.environ[path_var] = dir + os.pathsep + os.environ[path_var]
+
+del os, sys, path_var, dir
+
+
+def _exec_tool(tool):
+    import os, sys
+    from subprocess import Popen
+    tools_dir = os.path.dirname(__file__)
+    handle = Popen(sys.argv, executable=os.path.join(tools_dir, tool))
+    try:
+        try:
+            return handle.wait()
+        except KeyboardInterrupt:
+            # Give the program a chance to handle the signal gracefully.
+            return handle.wait()
+    except:
+        handle.kill()
+        handle.wait()
+        raise
+
+# Register all the executables in this directory as global functions.
+{0}
+"""
+
+
+def parse_dependencies_windows(data):
+    """ Parses the given output from dumpbin /dependents to determine the list
+    of dll's this executable file depends on. """
+
+    lines = data.splitlines()
+    li = 0
+    while li < len(lines):
+        line = lines[li]
+        li += 1
+        if line.find(' has the following dependencies') != -1:
+            break
+
+    if li < len(lines):
+        line = lines[li]
+        if line.strip() == '':
+            # Skip a blank line.
+            li += 1
+
+    # Now we're finding filenames, until the next blank line.
+    filenames = []
+    while li < len(lines):
+        line = lines[li]
+        li += 1
+        line = line.strip()
+        if line == '':
+            # We're done.
+            return filenames
+        filenames.append(line)
+
+    # At least we got some data.
+    return filenames
+
+
+def parse_dependencies_unix(data):
+    """ Parses the given output from otool -XL or ldd to determine the list of
+    libraries this executable file depends on. """
+
+    lines = data.splitlines()
+    filenames = []
+    for l in lines:
+        l = l.strip()
+        if l != "statically linked":
+            filenames.append(l.split(' ', 1)[0])
+    return filenames
+
+
+def scan_dependencies(pathname):
+    """ Checks the named file for DLL dependencies, and adds any appropriate
+    dependencies found into pluginDependencies and dependentFiles. """
+
+    if sys.platform == "darwin":
+        command = ['otool', '-XL', pathname]
+    elif sys.platform in ("win32", "cygwin"):
+        command = ['dumpbin', '/dependents', pathname]
+    else:
+        command = ['ldd', pathname]
+
+    output = subprocess.check_output(command, universal_newlines=True)
+    filenames = None
+
+    if sys.platform in ("win32", "cygwin"):
+        filenames = parse_dependencies_windows(output)
+    else:
+        filenames = parse_dependencies_unix(output)
+
+    if filenames is None:
+        sys.exit("Unable to determine dependencies from %s" % (pathname))
+
+    return filenames
+
+
+class WheelFile(object):
+    def __init__(self, name, version):
+        self.name = name
+        self.version = version
+
+        wheel_name = "{}-{}-{}-{}-{}.whl".format(
+            name, version, PY_VERSION, ABI_TAG, PLATFORM_TAG)
+
+        print("Writing %s" % (wheel_name))
+        self.zip_file = zipfile.ZipFile(wheel_name, 'w', zipfile.ZIP_DEFLATED)
+        self.records = []
+
+        # Used to locate dependency libraries.
+        self.lib_path = []
+        self.dep_paths = {}
+
+    def consider_add_dependency(self, target_path, dep, search_path=None):
+        """Considers adding a dependency library.
+        Returns the target_path if it was added, which may be different from
+        target_path if it was already added earlier, or None if it wasn't."""
+
+        if dep in self.dep_paths:
+            # Already considered this.
+            return self.dep_paths[dep]
+
+        self.dep_paths[dep] = None
+
+        if dep.lower().startswith("python"):
+            # Don't include the Python library.
+            return
+
+        source_path = None
+
+        if search_path is None:
+            search_path = self.lib_path
+
+        for lib_dir in search_path:
+            # Ignore static stuff.
+            path = os.path.join(lib_dir, dep)
+            if os.path.isfile(path):
+                source_path = os.path.normpath(path)
+                break
+
+        if not source_path:
+            # Couldn't find library in the panda3d lib dir.
+            #print("Ignoring %s" % (dep))
+            return
+
+        self.dep_paths[dep] = target_path
+        self.write_file(target_path, source_path)
+        return target_path
+
+    def write_file(self, target_path, source_path):
+        """Adds the given file to the .whl file."""
+
+        # If this is a .so file, we should set the rpath appropriately.
+        temp = None
+        ext = os.path.splitext(source_path)[1]
+        if ext in ('.so', '.dylib') or '.so.' in os.path.basename(source_path) or \
+            (not ext and is_executable(source_path)):
+            # Scan and add Unix dependencies.
+            deps = scan_dependencies(source_path)
+            for dep in deps:
+                # Only include dependencies with relative path.  Otherwise we
+                # end up overwriting system files like /lib/ld-linux.so.2!
+                # Yes, it happened to me.
+                if '/' not in dep:
+                    target_dep = os.path.dirname(target_path) + '/' + dep
+                    self.consider_add_dependency(target_dep, dep)
+
+            suffix = ''
+            if '.so' in os.path.basename(source_path):
+                suffix = '.so'
+            elif ext == '.dylib':
+                suffix = '.dylib'
+
+            temp = tempfile.NamedTemporaryFile(suffix=suffix, prefix='whl', delete=False)
+            temp.write(open(source_path, 'rb').read())
+            os.fchmod(temp.fileno(), os.fstat(temp.fileno()).st_mode | 0o111)
+            temp.close()
+
+            # Fix things like @loader_path/../lib references
+            if sys.platform == "darwin":
+                loader_path = [os.path.dirname(source_path)]
+                for dep in deps:
+                    if '@loader_path' not in dep:
+                        continue
+
+                    dep_path = dep.replace('@loader_path', '.')
+                    target_dep = os.path.dirname(target_path) + '/' + os.path.basename(dep)
+                    target_dep = self.consider_add_dependency(target_dep, dep_path, loader_path)
+                    if not target_dep:
+                        # It won't be included, so no use adjusting the path.
+                        continue
+
+                    new_dep = os.path.join('@loader_path', os.path.relpath(target_dep, os.path.dirname(target_path)))
+                    subprocess.call(["install_name_tool", "-change", dep, new_dep, temp.name])
+            else:
+                subprocess.call(["strip", "-s", temp.name])
+                subprocess.call(["patchelf", "--set-rpath", "$ORIGIN", temp.name])
+
+            source_path = temp.name
+
+        ext = ext.lower()
+        if ext in ('.dll', '.pyd', '.exe'):
+            # Scan and add Win32 dependencies.
+            for dep in scan_dependencies(source_path):
+                target_dep = os.path.dirname(target_path) + '/' + dep
+                self.consider_add_dependency(target_dep, dep)
+
+        # Calculate the SHA-256 hash and size.
+        sha = hashlib.sha256()
+        fp = open(source_path, 'rb')
+        size = 0
+        data = fp.read(1024 * 1024)
+        while data:
+            size += len(data)
+            sha.update(data)
+            data = fp.read(1024 * 1024)
+        fp.close()
+
+        # Save it in PEP-0376 format for writing out later.
+        digest = str(urlsafe_b64encode(sha.digest()))
+        digest = digest.rstrip('=')
+        self.records.append("{},sha256={},{}\n".format(target_path, digest, size))
+
+        if GetVerbose():
+            print("Adding %s from %s" % (target_path, source_path))
+        self.zip_file.write(source_path, target_path)
+
+        #if temp:
+        #    os.unlink(temp.name)
+
+    def write_file_data(self, target_path, source_data):
+        """Adds the given file from a string."""
+
+        sha = hashlib.sha256()
+        sha.update(source_data.encode())
+        digest = str(urlsafe_b64encode(sha.digest()))
+        digest = digest.rstrip('=')
+        self.records.append("{},sha256={},{}\n".format(target_path, digest, len(source_data)))
+
+        if GetVerbose():
+            print("Adding %s from data" % target_path)
+        self.zip_file.writestr(target_path, source_data)
+
+    def write_directory(self, target_dir, source_dir):
+        """Adds the given directory recursively to the .whl file."""
+
+        for root, dirs, files in os.walk(source_dir):
+            for file in files:
+                if os.path.splitext(file)[1] in EXCLUDE_EXT:
+                    continue
+
+                source_path = os.path.join(root, file)
+                target_path = os.path.join(target_dir, os.path.relpath(source_path, source_dir))
+                target_path = target_path.replace('\\', '/')
+                self.write_file(target_path, source_path)
+
+    def close(self):
+        # Write the RECORD file.
+        record_file = "{}-{}.dist-info/RECORD".format(self.name, self.version)
+        self.records.append(record_file + ",,\n")
+
+        self.zip_file.writestr(record_file, "".join(self.records))
+        self.zip_file.close()
+
+
+def makewheel(version, output_dir):
+    if sys.platform not in ("win32", "darwin") and not sys.platform.startswith("cygwin"):
+        if not LocateBinary("patchelf"):
+            raise Exception("patchelf is required when building a Linux wheel.")
+
+    # Global filepaths
+    panda3d_dir = join(output_dir, "panda3d")
+    pandac_dir = join(output_dir, "pandac")
+    direct_dir = join(output_dir, "direct")
+    models_dir = join(output_dir, "models")
+    etc_dir = join(output_dir, "etc")
+    bin_dir = join(output_dir, "bin")
+    if sys.platform == "win32":
+        libs_dir = join(output_dir, "bin")
+    else:
+        libs_dir = join(output_dir, "lib")
+    license_src = "LICENSE"
+    readme_src = "README.md"
+
+    # Update relevant METADATA entries
+    METADATA['version'] = version
+    version_classifiers = [
+        "Programming Language :: Python :: {}".format(*sys.version_info),
+        "Programming Language :: Python :: {}.{}".format(*sys.version_info),
+    ]
+    METADATA['classifiers'].extend(version_classifiers)
+
+    # Build out the metadata
+    details = METADATA["extensions"]["python.details"]
+    homepage = details["project_urls"]["Home"]
+    author = details["contacts"][0]["name"]
+    email = details["contacts"][0]["email"]
+    metadata = ''.join([
+        "Metadata-Version: {metadata_version}\n" \
+        "Name: {name}\n" \
+        "Version: {version}\n" \
+        "Summary: {summary}\n" \
+        "License: {license}\n".format(**METADATA),
+        "Home-page: {}\n".format(homepage),
+        "Author: {}\n".format(author),
+        "Author-email: {}\n".format(email),
+        "Platform: {}\n".format(PLATFORM_TAG),
+    ] + ["Classifier: {}\n".format(c) for c in METADATA['classifiers']])
+
+    # Zip it up and name it the right thing
+    whl = WheelFile('panda3d', version)
+    whl.lib_path = [libs_dir]
+
+    # Add the trees with Python modules.
+    whl.write_directory('direct', direct_dir)
+
+    # Write the panda3d tree.  We use a custom empty __init__ since the
+    # default one adds the bin directory to the PATH, which we don't have.
+    whl.write_file_data('panda3d/__init__.py', '')
+
+    ext_suffix = GetExtensionSuffix()
+
+    for file in os.listdir(panda3d_dir):
+        if file == '__init__.py':
+            pass
+        elif file.endswith(ext_suffix) or file.endswith('.py'):
+            source_path = os.path.join(panda3d_dir, file)
+
+            if file.endswith('.pyd') and PLATFORM_TAG.startswith('cygwin'):
+                # Rename it to .dll for cygwin Python to be able to load it.
+                target_path = 'panda3d/' + os.path.splitext(file)[0] + '.dll'
+            else:
+                target_path = 'panda3d/' + file
+            whl.write_file(target_path, source_path)
+
+    # Add plug-ins.
+    for lib in PLUGIN_LIBS:
+        plugin_name = 'lib' + lib
+        if sys.platform in ('win32', 'cygwin'):
+            plugin_name += '.dll'
+        elif sys.platform == 'darwin':
+            plugin_name += '.dylib'
+        else:
+            plugin_name += '.so'
+        plugin_path = os.path.join(libs_dir, plugin_name)
+        if os.path.isfile(plugin_path):
+            whl.write_file('panda3d/' + plugin_name, plugin_path)
+
+    # Add the pandac tree for backward compatibility.
+    for file in os.listdir(pandac_dir):
+        if file.endswith('.py'):
+            whl.write_file('pandac/' + file, os.path.join(pandac_dir, file))
+
+    # Add a panda3d-tools directory containing the executables.
+    entry_points = '[console_scripts]\n'
+    tools_init = ''
+    for file in os.listdir(bin_dir):
+        source_path = os.path.join(bin_dir, file)
+
+        if is_executable(source_path):
+            # Put the .exe files inside the panda3d-tools directory.
+            whl.write_file('panda3d_tools/' + file, source_path)
+
+            # Tell pip to create a wrapper script.
+            basename = os.path.splitext(file)[0]
+            funcname = basename.replace('-', '_')
+            entry_points += '{0} = panda3d_tools:{1}\n'.format(basename, funcname)
+            tools_init += '{0} = lambda: _exec_tool({1!r})\n'.format(funcname, file)
+
+    whl.write_file_data('panda3d_tools/__init__.py', PANDA3D_TOOLS_INIT.format(tools_init))
+
+    # Add the .data directory, containing additional files.
+    data_dir = 'panda3d-{}.data'.format(version)
+    #whl.write_directory(data_dir + '/data/etc', etc_dir)
+    #whl.write_directory(data_dir + '/data/models', models_dir)
+
+    # Actually, let's not.  That seems to install the files to the strangest
+    # places in the user's filesystem.  Let's instead put them in panda3d.
+    whl.write_directory('panda3d/etc', etc_dir)
+    whl.write_directory('panda3d/models', models_dir)
+
+    # Add the dist-info directory last.
+    info_dir = 'panda3d-{}.dist-info'.format(version)
+    whl.write_file_data(info_dir + '/entry_points.txt', entry_points)
+    whl.write_file_data(info_dir + '/metadata.json', json.dumps(METADATA, indent=4, separators=(',', ': ')))
+    whl.write_file_data(info_dir + '/METADATA', metadata)
+    whl.write_file_data(info_dir + '/WHEEL', WHEEL_DATA.format(PY_VERSION, ABI_TAG, PLATFORM_TAG))
+    whl.write_file(info_dir + '/LICENSE.txt', license_src)
+    whl.write_file(info_dir + '/README.md', readme_src)
+    whl.write_file_data(info_dir + '/top_level.txt', 'direct\npanda3d\npandac\npanda3d_tools\n')
+
+    whl.close()
+
+
+if __name__ == "__main__":
+    version = ParsePandaVersion("dtool/PandaVersion.pp")
+
+    parser = OptionParser()
+    parser.add_option('', '--version', dest = 'version', help = 'Panda3D version number (default: %s)' % (version), default = version)
+    parser.add_option('', '--outputdir', dest = 'outputdir', help = 'Makepanda\'s output directory (default: built)', default = 'built')
+    parser.add_option('', '--verbose', dest = 'verbose', help = 'Enable verbose output', action = 'store_true', default = False)
+    (options, args) = parser.parse_args()
+
+    SetVerbose(options.verbose)
+    makewheel(options.version, options.outputdir)