deploy-ng: change format of blob to be easily mappable by deploy-ng

direct/src/showutil/FreezeTool.py

@@ -1588,32 +1588,6 @@ class Freezer:
         return target
 
     def generateRuntimeFromStub(self, basename, stub_file):
-        def make_module_list_entry(code, offset, modulename, module):
-            size = len(code)
-            if getattr(module, "__path__", None):
-                # Indicate package by negative size
-                size = -size
-            modname = modulename.encode('ascii')
-            modnamelen = len(modulename)
-            return struct.pack(
-                '<b{0}sIi'.format(modnamelen),
-                modnamelen,
-                modname,
-                offset,
-                size
-            )
-
-        def make_forbidden_module_list_entry(modulename):
-            modname = modulename.encode('ascii')
-            modnamelen = len(modulename)
-            return struct.pack(
-                '<b{0}sIi'.format(modnamelen),
-                modnamelen,
-                modname,
-                0,
-                0
-            )
-
         # We must have a __main__ module to make an exe file.
         if not self.__writingModule('__main__'):
             message = "Can't generate an executable without a __main__ module."
@@ -1626,27 +1600,55 @@ class Freezer:
             target = basename
             modext = '.so'
 
+        address_offset = 0
+
+        # First gather up the strings for all the module names.
+        blob = b""
+        strings = set()
+
+        for moduleName, mdef in self.getModuleDefs():
+            strings.add(moduleName.encode('ascii'))
+
+        # Sort by length descending, allowing reuse of partial strings.
+        strings = sorted(strings, key=lambda str:-len(str))
+        string_offsets = {}
+
+        for string in strings:
+            # First check whether it's already in there; it could be part of
+            # a longer string.
+            offset = blob.find(string + b'\0')
+            if offset < 0:
+                offset = len(blob)
+                blob += string + b'\0'
+            string_offsets[string] = offset
+
         # Generate export table.
-        moduleBlob = bytes()
-        codeOffset = 0
         moduleList = []
 
         for moduleName, mdef in self.getModuleDefs():
             origName = mdef.moduleName
             if mdef.forbid:
                 # Explicitly disallow importing this module.
-                moduleList.append(make_forbidden_module_list_entry(moduleName))
+                moduleList.append((moduleName, 0, 0))
                 continue
 
+            # For whatever it's worth, align the code blocks.
+            if len(blob) & 3 != 0:
+                pad = (4 - (len(blob) & 3))
+                blob += b'\0' * pad
+
             assert not mdef.exclude
             # Allow importing this module.
             module = self.mf.modules.get(origName, None)
             code = getattr(module, "__code__", None)
             if code:
                 code = marshal.dumps(code)
-                moduleList.append(make_module_list_entry(code, codeOffset, moduleName, module))
+                size = len(code)
-                moduleBlob += code
+                if getattr(module, "__path__", None):
-                codeOffset += len(code)
+                    # Indicate package by negative size
+                    size = -size
+                moduleList.append((moduleName, len(blob), size))
+                blob += code
                 continue
 
             # This is a module with no associated Python code.  It is either
@@ -1664,21 +1666,52 @@ class Freezer:
                 code = 'import sys;del sys.modules["%s"];import sys,os,imp;imp.load_dynamic("%s",os.path.join(os.path.dirname(sys.executable), "%s%s"))' % (moduleName, moduleName, moduleName, modext)
                 code = compile(code, moduleName, 'exec')
                 code = marshal.dumps(code)
-                moduleList.append(make_module_list_entry(code, codeOffset, moduleName, module))
+                moduleList.append((moduleName, len(blob), len(code)))
-                moduleBlob += code
+                blob += code
-                codeOffset += len(code)
+
+        # Now compose the final blob.
+        if '64' in self.platform:
+            layout = '<QQixxxx'
+        else:
+            layout = '<IIi'
+
+        blob2 = bytes()
+        blobOffset = (len(moduleList) + 1) * struct.calcsize(layout)
+        blobOffset += address_offset
+
+        for moduleName, offset, size in moduleList:
+            encoded = moduleName.encode('ascii')
+            stringOffset = blobOffset + string_offsets[encoded]
+            if size != 0:
+                offset += blobOffset
+            blob2 += struct.pack(layout, stringOffset, offset, size)
+
+        blob2 += struct.pack(layout, 0, 0, 0)
+        assert len(blob2) + address_offset == blobOffset
+
+        blob = blob2 + blob
+        del blob2
+
+        # Total blob length should be aligned to 8 bytes.
+        if len(blob) & 7 != 0:
+            pad = (8 - (len(blob) & 7))
+            blob += b'\0' * pad
 
         # Build from pre-built binary stub
         with open(target, 'wb') as f:
             f.write(stub_file.read())
-            listoffset = f.tell()
+            offset = f.tell()
-            for mod in moduleList:
+
-                f.write(mod)
+            # Add padding to align to the page size, so it can be mmapped.
-            modsoffset = f.tell()
+            if offset % 4095 != 0:
-            f.write(moduleBlob)
+                pad = (4096 - (offset & 4095))
-            f.write(struct.pack('<I', listoffset))
+                f.write(b'\0' * pad)
-            f.write(struct.pack('<I', modsoffset))
+                offset += pad
-            f.write(struct.pack('<I', len(moduleList)))
+
+            f.write(blob)
+            # And tack on the offset of the blob to the end.
+            f.write(struct.pack('<Q', offset))
+
         os.chmod(target, 0o755)
         return target
 

pandatool/src/deploy-stub/deploy-stub.c

@@ -3,12 +3,16 @@
 #include "Python.h"
 #ifdef _WIN32
 #include "malloc.h"
+#else
+#include <sys/mman.h>
+#endif
 
 #ifdef __FreeBSD__
 #include <sys/sysctl.h>
 #endif
 
 #include <stdio.h>
+#include <stdint.h>
 
 #if PY_MAJOR_VERSION >= 3
 #include <locale.h>
@@ -34,8 +38,6 @@ static struct _inittab extensions[] = {
 #endif
 #endif
 
-static unsigned char *modblob = NULL;
-
 /* Main program */
 
 #ifdef WIN_UNICODE
@@ -159,8 +161,8 @@ int wmain(int argc, wchar_t *argv[]) {
 #else
 int main(int argc, char *argv[]) {
 #endif
-  struct _frozen *_PyImport_FrozenModules;
+  struct _frozen *blob, *moddef;
-  unsigned int listoff, modsoff, fsize, modsize, listsize, nummods, modidx;
+  uint64_t begin, end, size;
   int retval;
   FILE *runtime;
 
@@ -184,66 +186,46 @@ int main(int argc, char *argv[]) {
 #endif
 
   // Get offsets
-  fseek(runtime, -12, SEEK_END);
+  fseek(runtime, -8, SEEK_END);
-  fsize = ftell(runtime);
+  end = ftell(runtime);
-  fread(&listoff, 4, 1, runtime);
+  fread(&begin, 8, 1, runtime);
-  fread(&modsoff, 4, 1, runtime);
+  size = end - begin;
-  fread(&nummods, 4, 1, runtime);
-  modsize = fsize - modsoff;
-  listsize = modsoff - listoff;
 
-  // Read module blob
+  // mmap the section indicated by the offset (or malloc/fread on windows)
-  modblob = malloc(modsize);
+#ifdef _WIN32
-  fseek(runtime, modsoff, SEEK_SET);
+  blob = (struct _frozen *)malloc(size);
-  fread(modblob, modsize, 1, runtime);
+  assert(blob != NULL);
-
+  fseek(runtime, (long)begin, SEEK_SET);
-  // Read module list
+  fread(blob, size, 1, runtime);
-  _PyImport_FrozenModules = calloc(nummods + 1, sizeof(struct _frozen));
+#else
-  fseek(runtime, listoff, SEEK_SET);
+  blob = (struct _frozen *)mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fileno(runtime), begin);
-  for (modidx = 0; modidx < nummods; ++modidx) {
+  assert(blob != NULL);
-    struct _frozen *moddef = &_PyImport_FrozenModules[modidx];
+#endif
-    unsigned char name_size;
-    char *name = NULL, namebuf[256] = {0};
-    size_t nsize;
-    unsigned int codeptr;
-    int codesize;
-
-    // Name
-    fread(&name_size, 1, 1, runtime);
-    fread(namebuf, 1, name_size, runtime);
-    nsize = strlen(namebuf) + 1;
-    name = malloc(nsize);
-    memcpy(name, namebuf, nsize);
-    moddef->name = name;
-
-    // Pointer
-    fread(&codeptr, 4, 1, runtime);
-    moddef->code = modblob + codeptr;
-
-    // Size
-    fread(&codesize, 4, 1, runtime);
-    moddef->size = codesize;
-  }
 
   fclose(runtime);
 
-  // Uncomment this to print out the read in module list
+  // Offset the pointers in the table using the base mmap address.
-  //for (modidx = 0; modidx < nummods; ++modidx) {
+  moddef = blob;
-  //  struct _frozen *moddef = &_PyImport_FrozenModules[modidx];
+  while (moddef->name) {
-  //  printf("MOD: %s %p %d\n", moddef->name, (void*)moddef->code, moddef->size);
+    moddef->name = (char *)((uintptr_t)moddef->name + (uintptr_t)blob);
-  //}
+    if (moddef->code != 0) {
+      moddef->code = (unsigned char *)((uintptr_t)moddef->code + (uintptr_t)blob);
+    }
+    //printf("MOD: %s %p %d\n", moddef->name, (void*)moddef->code, moddef->size);
+    moddef++;
+  }
 
   // Run frozen application
-  PyImport_FrozenModules = _PyImport_FrozenModules;
+  PyImport_FrozenModules = blob;
   retval = Py_FrozenMain(argc, argv);
 
   // Free resources
-  free(modblob);
+#ifdef _WIN32
-  for (modidx = 0; modidx < nummods; ++modidx) {
+  free(blob);
-    struct _frozen *moddef = &_PyImport_FrozenModules[modidx];
+#else
-    free((void*)moddef->name);
+  munmap(blob, size);
-  }
+#endif
-  free(_PyImport_FrozenModules);
+  return retval;
 }
 
 #ifdef WIN_UNICODE