TheCloud/panda3d
1
0
Fork 0
mirror of https://github.com/panda3d/panda3d.git synced 2025-10-03 10:22:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

deploy-ng: change format of blob to be easily mappable by deploy-ng

Browse Source
This commit is contained in:
rdb 2017-09-19 22:08:41 +02:00
parent 1c018f5bdb
commit 2026879ac9
2 changed files with 113 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
direct/src/showutil/FreezeTool.py
View File

@ -1588,32 +1588,6 @@ class Freezer:
return target
def generateRuntimeFromStub(self, basename, stub_file):
def make_module_list_entry(code, offset, modulename, module):
size = len(code)
if getattr(module, "__path__", None):
# Indicate package by negative size
size = -size
modname = modulename.encode('ascii')
modnamelen = len(modulename)
return struct.pack(
'<b{0}sIi'.format(modnamelen),
modnamelen,
modname,
offset,
size
)
def make_forbidden_module_list_entry(modulename):
modname = modulename.encode('ascii')
modnamelen = len(modulename)
return struct.pack(
'<b{0}sIi'.format(modnamelen),
modnamelen,
modname,
0,
0
)
# We must have a __main__ module to make an exe file.
if not self.__writingModule('__main__'):
message = "Can't generate an executable without a __main__ module."
@ -1626,27 +1600,55 @@ class Freezer:
target = basename
modext = '.so'
address_offset = 0
# First gather up the strings for all the module names.
blob = b""
strings = set()
for moduleName, mdef in self.getModuleDefs():
strings.add(moduleName.encode('ascii'))
# Sort by length descending, allowing reuse of partial strings.
strings = sorted(strings, key=lambda str:-len(str))
string_offsets = {}
for string in strings:
# First check whether it's already in there; it could be part of
# a longer string.
offset = blob.find(string + b'\0')
if offset < 0:
offset = len(blob)
blob += string + b'\0'
string_offsets[string] = offset
# Generate export table.
moduleBlob = bytes()
codeOffset = 0
moduleList = []
for moduleName, mdef in self.getModuleDefs():
origName = mdef.moduleName
if mdef.forbid:
# Explicitly disallow importing this module.
moduleList.append(make_forbidden_module_list_entry(moduleName))
moduleList.append((moduleName, 0, 0))
continue
# For whatever it's worth, align the code blocks.
if len(blob) & 3 != 0:
pad = (4 - (len(blob) & 3))
blob += b'\0' * pad
assert not mdef.exclude
# Allow importing this module.
module = self.mf.modules.get(origName, None)
code = getattr(module, "__code__", None)
if code:
code = marshal.dumps(code)
moduleList.append(make_module_list_entry(code, codeOffset, moduleName, module))
moduleBlob += code
codeOffset += len(code)
size = len(code)
if getattr(module, "__path__", None):
# Indicate package by negative size
size = -size
moduleList.append((moduleName, len(blob), size))
blob += code
continue
# This is a module with no associated Python code. It is either
@ -1664,21 +1666,52 @@ class Freezer:
code = 'import sys;del sys.modules["%s"];import sys,os,imp;imp.load_dynamic("%s",os.path.join(os.path.dirname(sys.executable), "%s%s"))' % (moduleName, moduleName, moduleName, modext)
code = compile(code, moduleName, 'exec')
code = marshal.dumps(code)
moduleList.append(make_module_list_entry(code, codeOffset, moduleName, module))
moduleBlob += code
codeOffset += len(code)
moduleList.append((moduleName, len(blob), len(code)))
blob += code
# Now compose the final blob.
if '64' in self.platform:
layout = '<QQixxxx'
else:
layout = '<IIi'
blob2 = bytes()
blobOffset = (len(moduleList) + 1) * struct.calcsize(layout)
blobOffset += address_offset
for moduleName, offset, size in moduleList:
encoded = moduleName.encode('ascii')
stringOffset = blobOffset + string_offsets[encoded]
if size != 0:
offset += blobOffset
blob2 += struct.pack(layout, stringOffset, offset, size)
blob2 += struct.pack(layout, 0, 0, 0)
assert len(blob2) + address_offset == blobOffset
blob = blob2 + blob
del blob2
# Total blob length should be aligned to 8 bytes.
if len(blob) & 7 != 0:
pad = (8 - (len(blob) & 7))
blob += b'\0' * pad
# Build from pre-built binary stub
with open(target, 'wb') as f:
f.write(stub_file.read())
listoffset = f.tell()
for mod in moduleList:
f.write(mod)
modsoffset = f.tell()
f.write(moduleBlob)
f.write(struct.pack('<I', listoffset))
f.write(struct.pack('<I', modsoffset))
f.write(struct.pack('<I', len(moduleList)))
offset = f.tell()
# Add padding to align to the page size, so it can be mmapped.
if offset % 4095 != 0:
pad = (4096 - (offset & 4095))
f.write(b'\0' * pad)
offset += pad
f.write(blob)
# And tack on the offset of the blob to the end.
f.write(struct.pack('<Q', offset))
os.chmod(target, 0o755)
return target

92
pandatool/src/deploy-stub/deploy-stub.c
View File

@ -3,12 +3,16 @@
#include "Python.h"
#ifdef _WIN32
#include "malloc.h"
#else
#include <sys/mman.h>
#endif
#ifdef __FreeBSD__
#include <sys/sysctl.h>
#endif
#include <stdio.h>
#include <stdint.h>
#if PY_MAJOR_VERSION >= 3
#include <locale.h>
@ -34,8 +38,6 @@ static struct _inittab extensions[] = {
#endif
#endif
static unsigned char *modblob = NULL;
/* Main program */
#ifdef WIN_UNICODE
@ -159,8 +161,8 @@ int wmain(int argc, wchar_t *argv[]) {
#else
int main(int argc, char *argv[]) {
#endif
struct _frozen *_PyImport_FrozenModules;
unsigned int listoff, modsoff, fsize, modsize, listsize, nummods, modidx;
struct _frozen *blob, *moddef;
uint64_t begin, end, size;
int retval;
FILE *runtime;
@ -184,66 +186,46 @@ int main(int argc, char *argv[]) {
#endif
// Get offsets
fseek(runtime, -12, SEEK_END);
fsize = ftell(runtime);
fread(&listoff, 4, 1, runtime);
fread(&modsoff, 4, 1, runtime);
fread(&nummods, 4, 1, runtime);
modsize = fsize - modsoff;
listsize = modsoff - listoff;
fseek(runtime, -8, SEEK_END);
end = ftell(runtime);
fread(&begin, 8, 1, runtime);
size = end - begin;
// Read module blob
modblob = malloc(modsize);
fseek(runtime, modsoff, SEEK_SET);
fread(modblob, modsize, 1, runtime);
// Read module list
_PyImport_FrozenModules = calloc(nummods + 1, sizeof(struct _frozen));
fseek(runtime, listoff, SEEK_SET);
for (modidx = 0; modidx < nummods; ++modidx) {
struct _frozen *moddef = &_PyImport_FrozenModules[modidx];
unsigned char name_size;
char *name = NULL, namebuf[256] = {0};
size_t nsize;
unsigned int codeptr;
int codesize;
// Name
fread(&name_size, 1, 1, runtime);
fread(namebuf, 1, name_size, runtime);
nsize = strlen(namebuf) + 1;
name = malloc(nsize);
memcpy(name, namebuf, nsize);
moddef->name = name;
// Pointer
fread(&codeptr, 4, 1, runtime);
moddef->code = modblob + codeptr;
// Size
fread(&codesize, 4, 1, runtime);
moddef->size = codesize;
}
// mmap the section indicated by the offset (or malloc/fread on windows)
#ifdef _WIN32
blob = (struct _frozen *)malloc(size);
assert(blob != NULL);
fseek(runtime, (long)begin, SEEK_SET);
fread(blob, size, 1, runtime);
#else
blob = (struct _frozen *)mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fileno(runtime), begin);
assert(blob != NULL);
#endif
fclose(runtime);
// Uncomment this to print out the read in module list
//for (modidx = 0; modidx < nummods; ++modidx) {
// struct _frozen *moddef = &_PyImport_FrozenModules[modidx];
// printf("MOD: %s %p %d\n", moddef->name, (void*)moddef->code, moddef->size);
//}
// Offset the pointers in the table using the base mmap address.
moddef = blob;
while (moddef->name) {
moddef->name = (char *)((uintptr_t)moddef->name + (uintptr_t)blob);
if (moddef->code != 0) {
moddef->code = (unsigned char *)((uintptr_t)moddef->code + (uintptr_t)blob);
}
//printf("MOD: %s %p %d\n", moddef->name, (void*)moddef->code, moddef->size);
moddef++;
}
// Run frozen application
PyImport_FrozenModules = _PyImport_FrozenModules;
PyImport_FrozenModules = blob;
retval = Py_FrozenMain(argc, argv);
// Free resources
free(modblob);
for (modidx = 0; modidx < nummods; ++modidx) {
struct _frozen *moddef = &_PyImport_FrozenModules[modidx];
free((void*)moddef->name);
}
free(_PyImport_FrozenModules);
#ifdef _WIN32
free(blob);
#else
munmap(blob, size);
#endif
return retval;
}
#ifdef WIN_UNICODE