Add ability to produce .whl file

Based on original version by pennomi
Closes: #83
This commit is contained in:
rdb 2016-12-19 23:35:05 +01:00
parent 04819719fb
commit b2ccf6c0d2
2 changed files with 577 additions and 2 deletions

View File

@ -39,6 +39,7 @@ import sys
COMPILER=0
INSTALLER=0
WHEEL=0
GENMAN=0
COMPRESSOR="zlib"
THREADCOUNT=0
@ -124,6 +125,7 @@ def usage(problem):
print(" --verbose (print out more information)")
print(" --runtime (build a runtime build instead of an SDK build)")
print(" --installer (build an installer)")
print(" --wheel (build a pip-installable .whl)")
print(" --optimize X (optimization level can be 1,2,3,4)")
print(" --version X (set the panda version number)")
print(" --lzma (use lzma compression when building Windows installer)")
@ -159,13 +161,13 @@ def usage(problem):
os._exit(1)
def parseopts(args):
global INSTALLER,RTDIST,RUNTIME,GENMAN,DISTRIBUTOR,VERSION
global INSTALLER,WHEEL,RTDIST,RUNTIME,GENMAN,DISTRIBUTOR,VERSION
global COMPRESSOR,THREADCOUNT,OSXTARGET,OSX_ARCHS,HOST_URL
global DEBVERSION,RPMRELEASE,GIT_COMMIT,P3DSUFFIX,RTDIST_VERSION
global STRDXSDKVERSION, WINDOWS_SDK, MSVC_VERSION, BOOUSEINTELCOMPILER
longopts = [
"help","distributor=","verbose","runtime","osxtarget=",
"optimize=","everything","nothing","installer","rtdist","nocolor",
"optimize=","everything","nothing","installer","wheel","rtdist","nocolor",
"version=","lzma","no-python","threads=","outputdir=","override=",
"static","host=","debversion=","rpmrelease=","p3dsuffix=","rtdist-version=",
"directx-sdk=", "windows-sdk=", "msvc-version=", "clean", "use-icl",
@ -188,6 +190,7 @@ def parseopts(args):
if (option=="--help"): raise Exception
elif (option=="--optimize"): optimize=value
elif (option=="--installer"): INSTALLER=1
elif (option=="--wheel"): WHEEL=1
elif (option=="--verbose"): SetVerbose(True)
elif (option=="--distributor"): DISTRIBUTOR=value
elif (option=="--rtdist"): RTDIST=1
@ -416,9 +419,18 @@ if (RUNTIME):
if (INSTALLER and RTDIST):
exit("Cannot build an installer for the rtdist build!")
if (WHEEL and RUNTIME):
exit("Cannot build a wheel for the runtime build!")
if (WHEEL and RTDIST):
exit("Cannot build a wheel for the rtdist build!")
if (INSTALLER) and (PkgSkip("PYTHON")) and (not RUNTIME) and GetTarget() == 'windows':
exit("Cannot build installer on Windows without python")
if WHEEL and PkgSkip("PYTHON"):
exit("Cannot build wheel without Python")
if (RTDIST) and (PkgSkip("WX") and PkgSkip("FLTK")):
exit("Cannot build rtdist without wx or fltk")
@ -7239,6 +7251,11 @@ try:
MakeInstallerFreeBSD()
else:
exit("Do not know how to make an installer for this platform")
if WHEEL:
ProgressOutput(100.0, "Building wheel")
from makewheel import makewheel
makewheel(VERSION, GetOutputDir())
finally:
SaveDependencyCache()

558
makepanda/makewheel.py Normal file
View File

@ -0,0 +1,558 @@
"""
Generates a wheel (.whl) file from the output of makepanda.
Since the wheel requires special linking, this will only work if compiled with
the `--wheel` parameter.
"""
from __future__ import print_function, unicode_literals
from distutils.util import get_platform as get_dist
import json
import sys
import os
from os.path import join
import shutil
import zipfile
import hashlib
import tempfile
import subprocess
from sysconfig import get_config_var
from optparse import OptionParser
from makepandacore import ColorText, LocateBinary, ParsePandaVersion, GetExtensionSuffix, SetVerbose, GetVerbose
from base64 import urlsafe_b64encode
def get_platform():
p = get_dist().replace('-', '_').replace('.', '_')
#if "linux" in p:
# print(ColorText("red", "WARNING:") +
# " Linux-specific wheel files are not supported."
# " We will generate this wheel as a generic package instead.")
# return "any"
return p
def get_abi_tag():
if sys.version_info >= (3, 0):
soabi = get_config_var('SOABI')
if soabi and soabi.startswith('cpython-'):
return 'cp' + soabi.split('-')[1]
elif soabi:
return soabi.replace('.', '_').replace('-', '_')
soabi = 'cp%d%d' % (sys.version_info[:2])
debug_flag = get_config_var('Py_DEBUG')
if (debug_flag is None and hasattr(sys, 'gettotalrefcount')) or debug_flag:
soabi += 'd'
malloc_flag = get_config_var('WITH_PYMALLOC')
if malloc_flag is None or malloc_flag:
soabi += 'm'
if sys.version_info < (3, 3):
usize = get_config_var('Py_UNICODE_SIZE')
if (usize is None and sys.maxunicode == 0x10ffff) or usize == 4:
soabi += 'u'
return soabi
def is_exe_file(path):
return os.path.isfile(path) and path.lower().endswith('.exe')
def is_elf_file(path):
base = os.path.basename(path)
return os.path.isfile(path) and '.' not in base and \
open(path, 'rb').read(4) == b'\x7FELF'
def is_mach_o_file(path):
base = os.path.basename(path)
return os.path.isfile(path) and '.' not in base and \
open(path, 'rb').read(4) == b'\xCA\xFE\xBA\xBE'
if sys.platform in ('win32', 'cygwin'):
is_executable = is_exe_file
elif sys.platform == 'darwin':
is_executable = is_mach_o_file
else:
is_executable = is_elf_file
# Other global parameters
PY_VERSION = "cp{}{}".format(sys.version_info.major, sys.version_info.minor)
ABI_TAG = get_abi_tag()
PLATFORM_TAG = get_platform()
EXCLUDE_EXT = [".pyc", ".pyo", ".N", ".prebuilt", ".xcf", ".plist", ".vcproj", ".sln"]
# Plug-ins to install.
PLUGIN_LIBS = ["pandagl", "pandagles", "pandagles2", "p3ptloader", "p3assimp", "p3ffmpeg", "p3openal_audio", "p3fmod_audio"]
WHEEL_DATA = """Wheel-Version: 1.0
Generator: makepanda
Root-Is-Purelib: false
Tag: {}-{}-{}
"""
METADATA = {
"license": "BSD",
"name": "Panda3D",
"metadata_version": "2.0",
"generator": "makepanda",
"summary": "Panda3D is a game engine, a framework for 3D rendering and "
"game development for Python and C++ programs.",
"extensions": {
"python.details": {
"project_urls": {
"Home": "https://www.panda3d.org/"
},
"document_names": {
"license": "LICENSE.txt"
},
"contacts": [
{
"role": "author",
"email": "etc-panda3d@lists.andrew.cmu.edu",
"name": "Panda3D Team"
}
]
}
},
"classifiers": [
"Development Status :: 5 - Production/Stable",
"Intended Audience :: Developers",
"Intended Audience :: End Users/Desktop",
"License :: OSI Approved :: BSD License",
"Operating System :: OS Independent",
"Programming Language :: C++",
"Programming Language :: Python",
"Topic :: Games/Entertainment",
"Topic :: Multimedia",
"Topic :: Multimedia :: Graphics",
"Topic :: Multimedia :: Graphics :: 3D Rendering"
]
}
PANDA3D_TOOLS_INIT = """import os, sys
import panda3d
if sys.platform in ('win32', 'cygwin'):
path_var = 'PATH'
elif sys.platform == 'darwin':
path_var = 'DYLD_LIBRARY_PATH'
else:
path_var = 'LD_LIBRARY_PATH'
dir = os.path.dirname(panda3d.__file__)
del panda3d
if not os.environ.get(path_var):
os.environ[path_var] = dir
else:
os.environ[path_var] = dir + os.pathsep + os.environ[path_var]
del os, sys, path_var, dir
def _exec_tool(tool):
import os, sys
from subprocess import Popen
tools_dir = os.path.dirname(__file__)
handle = Popen(sys.argv, executable=os.path.join(tools_dir, tool))
try:
try:
return handle.wait()
except KeyboardInterrupt:
# Give the program a chance to handle the signal gracefully.
return handle.wait()
except:
handle.kill()
handle.wait()
raise
# Register all the executables in this directory as global functions.
{0}
"""
def parse_dependencies_windows(data):
""" Parses the given output from dumpbin /dependents to determine the list
of dll's this executable file depends on. """
lines = data.splitlines()
li = 0
while li < len(lines):
line = lines[li]
li += 1
if line.find(' has the following dependencies') != -1:
break
if li < len(lines):
line = lines[li]
if line.strip() == '':
# Skip a blank line.
li += 1
# Now we're finding filenames, until the next blank line.
filenames = []
while li < len(lines):
line = lines[li]
li += 1
line = line.strip()
if line == '':
# We're done.
return filenames
filenames.append(line)
# At least we got some data.
return filenames
def parse_dependencies_unix(data):
""" Parses the given output from otool -XL or ldd to determine the list of
libraries this executable file depends on. """
lines = data.splitlines()
filenames = []
for l in lines:
l = l.strip()
if l != "statically linked":
filenames.append(l.split(' ', 1)[0])
return filenames
def scan_dependencies(pathname):
""" Checks the named file for DLL dependencies, and adds any appropriate
dependencies found into pluginDependencies and dependentFiles. """
if sys.platform == "darwin":
command = ['otool', '-XL', pathname]
elif sys.platform in ("win32", "cygwin"):
command = ['dumpbin', '/dependents', pathname]
else:
command = ['ldd', pathname]
output = subprocess.check_output(command, universal_newlines=True)
filenames = None
if sys.platform in ("win32", "cygwin"):
filenames = parse_dependencies_windows(output)
else:
filenames = parse_dependencies_unix(output)
if filenames is None:
sys.exit("Unable to determine dependencies from %s" % (pathname))
return filenames
class WheelFile(object):
def __init__(self, name, version):
self.name = name
self.version = version
wheel_name = "{}-{}-{}-{}-{}.whl".format(
name, version, PY_VERSION, ABI_TAG, PLATFORM_TAG)
print("Writing %s" % (wheel_name))
self.zip_file = zipfile.ZipFile(wheel_name, 'w', zipfile.ZIP_DEFLATED)
self.records = []
# Used to locate dependency libraries.
self.lib_path = []
self.dep_paths = {}
def consider_add_dependency(self, target_path, dep, search_path=None):
"""Considers adding a dependency library.
Returns the target_path if it was added, which may be different from
target_path if it was already added earlier, or None if it wasn't."""
if dep in self.dep_paths:
# Already considered this.
return self.dep_paths[dep]
self.dep_paths[dep] = None
if dep.lower().startswith("python"):
# Don't include the Python library.
return
source_path = None
if search_path is None:
search_path = self.lib_path
for lib_dir in search_path:
# Ignore static stuff.
path = os.path.join(lib_dir, dep)
if os.path.isfile(path):
source_path = os.path.normpath(path)
break
if not source_path:
# Couldn't find library in the panda3d lib dir.
#print("Ignoring %s" % (dep))
return
self.dep_paths[dep] = target_path
self.write_file(target_path, source_path)
return target_path
def write_file(self, target_path, source_path):
"""Adds the given file to the .whl file."""
# If this is a .so file, we should set the rpath appropriately.
temp = None
ext = os.path.splitext(source_path)[1]
if ext in ('.so', '.dylib') or '.so.' in os.path.basename(source_path) or \
(not ext and is_executable(source_path)):
# Scan and add Unix dependencies.
deps = scan_dependencies(source_path)
for dep in deps:
# Only include dependencies with relative path. Otherwise we
# end up overwriting system files like /lib/ld-linux.so.2!
# Yes, it happened to me.
if '/' not in dep:
target_dep = os.path.dirname(target_path) + '/' + dep
self.consider_add_dependency(target_dep, dep)
suffix = ''
if '.so' in os.path.basename(source_path):
suffix = '.so'
elif ext == '.dylib':
suffix = '.dylib'
temp = tempfile.NamedTemporaryFile(suffix=suffix, prefix='whl', delete=False)
temp.write(open(source_path, 'rb').read())
os.fchmod(temp.fileno(), os.fstat(temp.fileno()).st_mode | 0o111)
temp.close()
# Fix things like @loader_path/../lib references
if sys.platform == "darwin":
loader_path = [os.path.dirname(source_path)]
for dep in deps:
if '@loader_path' not in dep:
continue
dep_path = dep.replace('@loader_path', '.')
target_dep = os.path.dirname(target_path) + '/' + os.path.basename(dep)
target_dep = self.consider_add_dependency(target_dep, dep_path, loader_path)
if not target_dep:
# It won't be included, so no use adjusting the path.
continue
new_dep = os.path.join('@loader_path', os.path.relpath(target_dep, os.path.dirname(target_path)))
subprocess.call(["install_name_tool", "-change", dep, new_dep, temp.name])
else:
subprocess.call(["strip", "-s", temp.name])
subprocess.call(["patchelf", "--set-rpath", "$ORIGIN", temp.name])
source_path = temp.name
ext = ext.lower()
if ext in ('.dll', '.pyd', '.exe'):
# Scan and add Win32 dependencies.
for dep in scan_dependencies(source_path):
target_dep = os.path.dirname(target_path) + '/' + dep
self.consider_add_dependency(target_dep, dep)
# Calculate the SHA-256 hash and size.
sha = hashlib.sha256()
fp = open(source_path, 'rb')
size = 0
data = fp.read(1024 * 1024)
while data:
size += len(data)
sha.update(data)
data = fp.read(1024 * 1024)
fp.close()
# Save it in PEP-0376 format for writing out later.
digest = str(urlsafe_b64encode(sha.digest()))
digest = digest.rstrip('=')
self.records.append("{},sha256={},{}\n".format(target_path, digest, size))
if GetVerbose():
print("Adding %s from %s" % (target_path, source_path))
self.zip_file.write(source_path, target_path)
#if temp:
# os.unlink(temp.name)
def write_file_data(self, target_path, source_data):
"""Adds the given file from a string."""
sha = hashlib.sha256()
sha.update(source_data.encode())
digest = str(urlsafe_b64encode(sha.digest()))
digest = digest.rstrip('=')
self.records.append("{},sha256={},{}\n".format(target_path, digest, len(source_data)))
if GetVerbose():
print("Adding %s from data" % target_path)
self.zip_file.writestr(target_path, source_data)
def write_directory(self, target_dir, source_dir):
"""Adds the given directory recursively to the .whl file."""
for root, dirs, files in os.walk(source_dir):
for file in files:
if os.path.splitext(file)[1] in EXCLUDE_EXT:
continue
source_path = os.path.join(root, file)
target_path = os.path.join(target_dir, os.path.relpath(source_path, source_dir))
target_path = target_path.replace('\\', '/')
self.write_file(target_path, source_path)
def close(self):
# Write the RECORD file.
record_file = "{}-{}.dist-info/RECORD".format(self.name, self.version)
self.records.append(record_file + ",,\n")
self.zip_file.writestr(record_file, "".join(self.records))
self.zip_file.close()
def makewheel(version, output_dir):
if sys.platform not in ("win32", "darwin") and not sys.platform.startswith("cygwin"):
if not LocateBinary("patchelf"):
raise Exception("patchelf is required when building a Linux wheel.")
# Global filepaths
panda3d_dir = join(output_dir, "panda3d")
pandac_dir = join(output_dir, "pandac")
direct_dir = join(output_dir, "direct")
models_dir = join(output_dir, "models")
etc_dir = join(output_dir, "etc")
bin_dir = join(output_dir, "bin")
if sys.platform == "win32":
libs_dir = join(output_dir, "bin")
else:
libs_dir = join(output_dir, "lib")
license_src = "LICENSE"
readme_src = "README.md"
# Update relevant METADATA entries
METADATA['version'] = version
version_classifiers = [
"Programming Language :: Python :: {}".format(*sys.version_info),
"Programming Language :: Python :: {}.{}".format(*sys.version_info),
]
METADATA['classifiers'].extend(version_classifiers)
# Build out the metadata
details = METADATA["extensions"]["python.details"]
homepage = details["project_urls"]["Home"]
author = details["contacts"][0]["name"]
email = details["contacts"][0]["email"]
metadata = ''.join([
"Metadata-Version: {metadata_version}\n" \
"Name: {name}\n" \
"Version: {version}\n" \
"Summary: {summary}\n" \
"License: {license}\n".format(**METADATA),
"Home-page: {}\n".format(homepage),
"Author: {}\n".format(author),
"Author-email: {}\n".format(email),
"Platform: {}\n".format(PLATFORM_TAG),
] + ["Classifier: {}\n".format(c) for c in METADATA['classifiers']])
# Zip it up and name it the right thing
whl = WheelFile('panda3d', version)
whl.lib_path = [libs_dir]
# Add the trees with Python modules.
whl.write_directory('direct', direct_dir)
# Write the panda3d tree. We use a custom empty __init__ since the
# default one adds the bin directory to the PATH, which we don't have.
whl.write_file_data('panda3d/__init__.py', '')
ext_suffix = GetExtensionSuffix()
for file in os.listdir(panda3d_dir):
if file == '__init__.py':
pass
elif file.endswith(ext_suffix) or file.endswith('.py'):
source_path = os.path.join(panda3d_dir, file)
if file.endswith('.pyd') and PLATFORM_TAG.startswith('cygwin'):
# Rename it to .dll for cygwin Python to be able to load it.
target_path = 'panda3d/' + os.path.splitext(file)[0] + '.dll'
else:
target_path = 'panda3d/' + file
whl.write_file(target_path, source_path)
# Add plug-ins.
for lib in PLUGIN_LIBS:
plugin_name = 'lib' + lib
if sys.platform in ('win32', 'cygwin'):
plugin_name += '.dll'
elif sys.platform == 'darwin':
plugin_name += '.dylib'
else:
plugin_name += '.so'
plugin_path = os.path.join(libs_dir, plugin_name)
if os.path.isfile(plugin_path):
whl.write_file('panda3d/' + plugin_name, plugin_path)
# Add the pandac tree for backward compatibility.
for file in os.listdir(pandac_dir):
if file.endswith('.py'):
whl.write_file('pandac/' + file, os.path.join(pandac_dir, file))
# Add a panda3d-tools directory containing the executables.
entry_points = '[console_scripts]\n'
tools_init = ''
for file in os.listdir(bin_dir):
source_path = os.path.join(bin_dir, file)
if is_executable(source_path):
# Put the .exe files inside the panda3d-tools directory.
whl.write_file('panda3d_tools/' + file, source_path)
# Tell pip to create a wrapper script.
basename = os.path.splitext(file)[0]
funcname = basename.replace('-', '_')
entry_points += '{0} = panda3d_tools:{1}\n'.format(basename, funcname)
tools_init += '{0} = lambda: _exec_tool({1!r})\n'.format(funcname, file)
whl.write_file_data('panda3d_tools/__init__.py', PANDA3D_TOOLS_INIT.format(tools_init))
# Add the .data directory, containing additional files.
data_dir = 'panda3d-{}.data'.format(version)
#whl.write_directory(data_dir + '/data/etc', etc_dir)
#whl.write_directory(data_dir + '/data/models', models_dir)
# Actually, let's not. That seems to install the files to the strangest
# places in the user's filesystem. Let's instead put them in panda3d.
whl.write_directory('panda3d/etc', etc_dir)
whl.write_directory('panda3d/models', models_dir)
# Add the dist-info directory last.
info_dir = 'panda3d-{}.dist-info'.format(version)
whl.write_file_data(info_dir + '/entry_points.txt', entry_points)
whl.write_file_data(info_dir + '/metadata.json', json.dumps(METADATA, indent=4, separators=(',', ': ')))
whl.write_file_data(info_dir + '/METADATA', metadata)
whl.write_file_data(info_dir + '/WHEEL', WHEEL_DATA.format(PY_VERSION, ABI_TAG, PLATFORM_TAG))
whl.write_file(info_dir + '/LICENSE.txt', license_src)
whl.write_file(info_dir + '/README.md', readme_src)
whl.write_file_data(info_dir + '/top_level.txt', 'direct\npanda3d\npandac\npanda3d_tools\n')
whl.close()
if __name__ == "__main__":
version = ParsePandaVersion("dtool/PandaVersion.pp")
parser = OptionParser()
parser.add_option('', '--version', dest = 'version', help = 'Panda3D version number (default: %s)' % (version), default = version)
parser.add_option('', '--outputdir', dest = 'outputdir', help = 'Makepanda\'s output directory (default: built)', default = 'built')
parser.add_option('', '--verbose', dest = 'verbose', help = 'Enable verbose output', action = 'store_true', default = False)
(options, args) = parser.parse_args()
SetVerbose(options.verbose)
makewheel(options.version, options.outputdir)