From 0b53355347ae7d49ea80ce8e5780f34eba634bfb Mon Sep 17 00:00:00 2001 From: rdb Date: Mon, 18 Jan 2021 02:34:16 +0100 Subject: [PATCH 1/9] interrogate: respect SOURCE_DATE_EPOCH setting for file identifiers This can be used to ensure that the build is bit-for-bit reproducible. See https://reproducible-builds.org/docs/source-date-epoch/ --- dtool/src/interrogate/interrogate.cxx | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dtool/src/interrogate/interrogate.cxx b/dtool/src/interrogate/interrogate.cxx index e3cab85a6b..162621a580 100644 --- a/dtool/src/interrogate/interrogate.cxx +++ b/dtool/src/interrogate/interrogate.cxx @@ -544,7 +544,15 @@ main(int argc, char **argv) { // Make up a file identifier. This is just some bogus number that should be // the same in both the compiled-in code and in the database, so we can // check synchronicity at load time. - int file_identifier = time(nullptr); + // We allow overriding this value by setting SOURCE_DATE_EPOCH to support + // reproducible builds. + int file_identifier; + const char *source_date_epoch = getenv("SOURCE_DATE_EPOCH"); + if (source_date_epoch != nullptr && source_date_epoch[0] != 0) { + file_identifier = atoi(source_date_epoch); + } else { + file_identifier = time(nullptr); + } InterrogateModuleDef *def = builder.make_module_def(file_identifier); pofstream * the_output_include = nullptr; From 3e1d98c10523a85b9ccd5038c4ed51ef0ec9d70b Mon Sep 17 00:00:00 2001 From: rdb Date: Mon, 18 Jan 2021 14:15:55 +0100 Subject: [PATCH 2/9] multify: Respect SOURCE_DATE_EPOCH variable when used from command-line That said, we should probably encourage the use of -T0 (which doesn't write out timestamps to begin with). --- panda/src/downloadertools/multify.cxx | 18 ++++++++++++++++++ panda/src/express/multifile.I | 11 +++++++++++ panda/src/express/multifile.h | 1 + 3 files changed, 30 insertions(+) diff --git a/panda/src/downloadertools/multify.cxx b/panda/src/downloadertools/multify.cxx index bea4c5bedd..d649992b40 100644 --- a/panda/src/downloadertools/multify.cxx +++ b/panda/src/downloadertools/multify.cxx @@ -57,6 +57,7 @@ string dont_compress_str = "jpg,png,mp3,ogg"; // Default text extensions. May be overridden with -X. string text_ext_str = "txt"; +time_t source_date_epoch = (time_t)-1; bool got_record_timestamp_flag = false; bool record_timestamp_flag = true; @@ -430,6 +431,12 @@ add_files(const vector_string ¶ms) { needs_repack = true; } + if (multifile->get_record_timestamp() && source_date_epoch != (time_t)-1) { + if (multifile->get_timestamp() > source_date_epoch) { + multifile->set_timestamp(source_date_epoch); + } + } + if (needs_repack) { if (!multifile->repack()) { cerr << "Failed to write " << multifile_name << ".\n"; @@ -533,6 +540,12 @@ kill_files(const vector_string ¶ms) { } } + if (multifile->get_record_timestamp() && source_date_epoch != (time_t)-1) { + if (multifile->get_timestamp() > source_date_epoch) { + multifile->set_timestamp(source_date_epoch); + } + } + bool okflag = true; if (multifile->needs_repack()) { @@ -779,6 +792,11 @@ main(int argc, char **argv) { } } + const char *source_date_epoch_str = getenv("SOURCE_DATE_EPOCH"); + if (source_date_epoch_str != nullptr && source_date_epoch_str[0] != 0) { + source_date_epoch = (time_t)strtoll(source_date_epoch_str, nullptr, 10); + } + extern char *optarg; extern int optind; static const char *optflags = "crutxkvz123456789Z:T:X:S:f:OC:ep:P:F:h"; diff --git a/panda/src/express/multifile.I b/panda/src/express/multifile.I index 128eb5297e..21462cc467 100644 --- a/panda/src/express/multifile.I +++ b/panda/src/express/multifile.I @@ -67,6 +67,17 @@ get_timestamp() const { return _timestamp; } +/** + * Changes the overall mudification timestamp of the multifile. Note that this + * will be reset to the current time every time you modify a subfile. + * Only set this if you know what you are doing! + */ +INLINE void Multifile:: +set_timestamp(time_t timestamp) { + _timestamp = timestamp; + _timestamp_dirty = true; +} + /** * Sets the flag indicating whether timestamps should be recorded within the * Multifile or not. The default is true, indicating the Multifile will diff --git a/panda/src/express/multifile.h b/panda/src/express/multifile.h index 540bf434d2..99b5242127 100644 --- a/panda/src/express/multifile.h +++ b/panda/src/express/multifile.h @@ -59,6 +59,7 @@ PUBLISHED: INLINE bool needs_repack() const; INLINE time_t get_timestamp() const; + INLINE void set_timestamp(time_t timestamp); INLINE void set_record_timestamp(bool record_timestamp); INLINE bool get_record_timestamp() const; From 6520b68c2cfdb2d143a7626531024259fd63ca3f Mon Sep 17 00:00:00 2001 From: rdb Date: Mon, 18 Jan 2021 14:19:39 +0100 Subject: [PATCH 3/9] progbase: respect SOURCE_DATE_EPOCH in -write-man option --- pandatool/src/progbase/programBase.cxx | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/pandatool/src/progbase/programBase.cxx b/pandatool/src/progbase/programBase.cxx index a509e93edf..6feab7d997 100644 --- a/pandatool/src/progbase/programBase.cxx +++ b/pandatool/src/progbase/programBase.cxx @@ -201,14 +201,26 @@ write_man_page(std::ostream &out) { // Generate a date string for inclusion into the footer. char date_str[256]; date_str[0] = 0; - time_t current_time = time(nullptr); + time_t current_time; + tm *today = nullptr; - if (current_time != (time_t) -1) { - tm *today = localtime(¤t_time); - if (today == nullptr || 0 == strftime(date_str, 256, "%d %B %Y", today)) { - date_str[0] = 0; + // This variable overrides the time we write to the footer. + const char *source_date_epoch = getenv("SOURCE_DATE_EPOCH"); + if (source_date_epoch == nullptr || source_date_epoch[0] == 0 || + (current_time = (time_t)strtoll(source_date_epoch, nullptr, 10)) <= 0) { + current_time = time(nullptr); + if (current_time != (time_t)-1) { + today = localtime(¤t_time); } } + else { + // Format as UTC to avoid inconsistency being introduced due to timezones. + today = gmtime(¤t_time); + } + + if (today == nullptr || 0 == strftime(date_str, 256, "%d %B %Y", today)) { + date_str[0] = 0; + } out << " 1 \"" << date_str << "\" \"" << PandaSystem::get_version_string() << "\" Panda3D\n"; From 54638bfc10bd766563830adaac118a4e55b4b52b Mon Sep 17 00:00:00 2001 From: rdb Date: Mon, 18 Jan 2021 14:16:59 +0100 Subject: [PATCH 4/9] dtoolutil: Allow overriding PandaSystem::get_build_date() This is useful to create bit-for-bit reproducible builds. In the buildbots, we set it to the timestamp of the latest commit. --- dtool/src/dtoolutil/pandaSystem.cxx | 4 ++++ makepanda/makepanda.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/dtool/src/dtoolutil/pandaSystem.cxx b/dtool/src/dtoolutil/pandaSystem.cxx index 667d08b8d9..fa09e065ca 100644 --- a/dtool/src/dtoolutil/pandaSystem.cxx +++ b/dtool/src/dtoolutil/pandaSystem.cxx @@ -268,7 +268,11 @@ get_compiler() { */ string PandaSystem:: get_build_date() { +#ifdef PANDA_BUILD_DATE_STR + return PANDA_BUILD_DATE_STR; +#else return __DATE__ " " __TIME__; +#endif } /** diff --git a/makepanda/makepanda.py b/makepanda/makepanda.py index e8391482ff..6f65cc0253 100755 --- a/makepanda/makepanda.py +++ b/makepanda/makepanda.py @@ -3001,6 +3001,14 @@ def CreatePandaVersionFiles(): if GIT_COMMIT: pandaversion_h += "\n#define PANDA_GIT_COMMIT_STR \"%s\"\n" % (GIT_COMMIT) + # Allow creating a deterministic build by setting this. + source_date = os.environ.get("SOURCE_DATE_EPOCH") + if source_date: + # This matches the GCC / Clang format for __DATE__ __TIME__ + source_date = time.gmtime(int(source_date)) + source_date = time.strftime('%b %e %Y %H:%M:%S', source_date) + pandaversion_h += "\n#define PANDA_BUILD_DATE_STR \"%s\"\n" % (source_date) + if not RUNTIME: checkpandaversion_cxx = CHECKPANDAVERSION_CXX.replace("$VERSION1",str(version1)) checkpandaversion_cxx = checkpandaversion_cxx.replace("$VERSION2",str(version2)) From 68daa238b1a9a756eb4ae93d5eeab220619a8741 Mon Sep 17 00:00:00 2001 From: rdb Date: Mon, 18 Jan 2021 17:55:55 +0100 Subject: [PATCH 5/9] dist: Add some determinism support to bdist_apps It's necessary to set PYTHONHASHSEED=0 as well as SOURCE_DATE_EPOCH for deterministic compilation, and moreover, the generated zip files do still have timestamps in them. --- direct/src/dist/FreezeTool.py | 6 +++--- direct/src/dist/commands.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/direct/src/dist/FreezeTool.py b/direct/src/dist/FreezeTool.py index 8cc07aaadf..66f638d7e7 100644 --- a/direct/src/dist/FreezeTool.py +++ b/direct/src/dist/FreezeTool.py @@ -958,7 +958,7 @@ class Freezer: # Scan the directory, looking for .py files. modules = [] - for basename in os.listdir(pathname): + for basename in sorted(os.listdir(pathname)): if basename.endswith('.py') and basename != '__init__.py': modules.append(basename[:-3]) @@ -992,7 +992,7 @@ class Freezer: modulePath = self.getModulePath(topName) if modulePath: for dirname in modulePath: - for basename in os.listdir(dirname): + for basename in sorted(os.listdir(dirname)): if os.path.exists(os.path.join(dirname, basename, '__init__.py')): parentName = '%s.%s' % (topName, basename) newParentName = '%s.%s' % (newTopName, basename) @@ -2587,7 +2587,7 @@ class PandaModuleFinder(modulefinder.ModuleFinder): except OSError: self.msg(2, "can't list directory", dir) continue - for name in names: + for name in sorted(names): mod = None for suff in self.suffixes: n = len(suff) diff --git a/direct/src/dist/commands.py b/direct/src/dist/commands.py index 9d7b193cfd..633931d8fa 100644 --- a/direct/src/dist/commands.py +++ b/direct/src/dist/commands.py @@ -1045,6 +1045,7 @@ class build_apps(setuptools.Command): rootdir = os.getcwd() for dirname, subdirlist, filelist in os.walk(rootdir): + subdirlist.sort() dirpath = os.path.relpath(dirname, rootdir) if skip_directory(dirpath): self.announce('skipping directory {}'.format(dirpath)) @@ -1414,7 +1415,8 @@ class bdist_apps(setuptools.Command): zf.write(build_dir, base_dir) for dirpath, dirnames, filenames in os.walk(build_dir): - for name in sorted(dirnames): + dirnames.sort() + for name in dirnames: path = os.path.normpath(os.path.join(dirpath, name)) zf.write(path, path.replace(build_dir, base_dir, 1)) for name in filenames: @@ -1429,16 +1431,39 @@ class bdist_apps(setuptools.Command): build_cmd = self.get_finalized_command('build_apps') binary_names = list(build_cmd.console_apps.keys()) + list(build_cmd.gui_apps.keys()) + source_date = os.environ.get('SOURCE_DATE_EPOCH', '').strip() + if source_date: + max_mtime = int(source_date) + else: + max_mtime = None + def tarfilter(tarinfo): if tarinfo.isdir() or os.path.basename(tarinfo.name) in binary_names: tarinfo.mode = 0o755 else: tarinfo.mode = 0o644 + + # This isn't interesting information to retain for distribution. + tarinfo.uid = 0 + tarinfo.gid = 0 + tarinfo.uname = "" + tarinfo.gname = "" + + if max_mtime is not None and tarinfo.mtime >= max_mtime: + tarinfo.mtime = max_mtime + return tarinfo - with tarfile.open('{}.tar.{}'.format(basename, tar_compression), 'w|{}'.format(tar_compression)) as tf: + filename = '{}.tar.{}'.format(basename, tar_compression) + with tarfile.open(filename, 'w|{}'.format(tar_compression)) as tf: tf.add(build_dir, base_dir, filter=tarfilter) + if tar_compression == 'gz' and max_mtime is not None: + # Python provides no elegant way to overwrite the gzip timestamp. + with open(filename, 'r+b') as fp: + fp.seek(4) + fp.write(struct.pack(" Date: Mon, 18 Jan 2021 19:03:00 +0100 Subject: [PATCH 6/9] mathutil: Fix scaling BoundingSphere to infinite causing assertions This error occurs when a BoundingSphere with a large radius is scaled by an even larger radius such that the radius becomes infinite. In this case, the BoundingSphere should be properly marked as infinite so that it behaves properly (and doesn't cause other assertions down the line). --- panda/src/mathutil/boundingSphere.cxx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/panda/src/mathutil/boundingSphere.cxx b/panda/src/mathutil/boundingSphere.cxx index 32e48adad9..beb6cb6309 100644 --- a/panda/src/mathutil/boundingSphere.cxx +++ b/panda/src/mathutil/boundingSphere.cxx @@ -112,6 +112,10 @@ xform(const LMatrix4 &mat) { // Transform the center _center = _center * mat; + + if (cinf(_radius)) { + set_infinite(); + } } } From ef6aa9d6caf0e4f68776ab15df4f5954f01419a9 Mon Sep 17 00:00:00 2001 From: rdb Date: Mon, 18 Jan 2021 19:05:45 +0100 Subject: [PATCH 7/9] directtools: Fix repeated selections causing scaling node to get huge This appears to be a regression from 0fe56bd0a980e0791d129498e2eb60ade6a0506d, but I can't be sure. Before this fix, repeated clicks of an object would cause the scaling handles to get larger and larger, until eventually causing NaN assertions. --- direct/src/directtools/DirectManipulation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/direct/src/directtools/DirectManipulation.py b/direct/src/directtools/DirectManipulation.py index d7aeb1ee3b..b6ca76802c 100644 --- a/direct/src/directtools/DirectManipulation.py +++ b/direct/src/directtools/DirectManipulation.py @@ -1368,7 +1368,7 @@ class ObjectHandles(NodePath, DirectObject): self.setScalingFactor(1) def setScalingFactor(self, scaleFactor): - self.ohScalingFactor = self.ohScalingFactor * scaleFactor + self.ohScalingFactor = scaleFactor self.scalingNode.setScale(self.ohScalingFactor * self.directScalingFactor) def getScalingFactor(self): From a270a55ccd88eecf476c33f671e014ca988b4650 Mon Sep 17 00:00:00 2001 From: rdb Date: Mon, 18 Jan 2021 23:30:18 +0100 Subject: [PATCH 8/9] dist: Add ignoreImports mechanism, prevents every app including numpy Apparently a host of thirdparty packages currently get included by default, such as importlib.metadata -> toml -> numpy, and this is getting rather out of hand. The ignoreImports mechanism provides a way for us to flag certain imports as being optional dependencies. Also added is various "builtins" imports in Python 2.7 (which are all under version checks and would otherwise lead to the PyPI "builtins" package being included, which would pull in "future", etc.) --- direct/src/dist/FreezeTool.py | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/direct/src/dist/FreezeTool.py b/direct/src/dist/FreezeTool.py index 66f638d7e7..31dd818fb8 100644 --- a/direct/src/dist/FreezeTool.py +++ b/direct/src/dist/FreezeTool.py @@ -92,6 +92,41 @@ else: hiddenImports['matplotlib.backends._backend_tk'] = ['Tkinter'] +# These are modules that import other modules but shouldn't pick them up as +# dependencies (usually because they are optional). This prevents picking up +# unwanted dependencies. +ignoreImports = { + 'direct.showbase.PythonUtil': ['pstats', 'profile'], + + 'toml.encoder': ['numpy'], +} + +if sys.version_info >= (3, 8): + # importlib.metadata is a "provisional" module introduced in Python 3.8 that + # conditionally pulls in dependency-rich packages like "email" and "pep517" + # (the latter of which is a thirdparty package!) But it's only imported in + # one obscure corner, so we don't want to pull it in by default. + ignoreImports['importlib._bootstrap_external'] = ['importlib.metadata'] + ignoreImports['importlib.metadata'] = ['pep517'] + +if sys.version_info < (3, 0): + # Include everything that we know conditionally imports the "builtins" + # module in Python 3 only, because otherwise it would cause the Python 2.7 + # package "builtins" to be included as a dependency. + ignoreImports.update({ + 'direct.p3d.AppRunner': ['builtins'], + 'direct.showbase.ContainerLeakDetector': ['builtins'], + 'direct.showbase.LeakDetectors': ['builtins'], + 'direct.showbase.MessengerLeakDetector': ['builtins'], + 'direct.showbase.ObjectReport': ['builtins'], + 'direct.showbase.ProfileSession': ['builtins'], + 'direct.showbase.PythonUtil': ['builtins'] + ignoreImports['direct.showbase.PythonUtil'], + 'direct.showbase.ShowBase': ['builtins'], + 'direct.showbase.ShowBaseGlobal': ['builtins'], + 'py._builtin': ['builtins'], + }) + + # These are overrides for specific modules. overrideModules = { # Used by the warnings module, among others, to get line numbers. Since @@ -2409,6 +2444,11 @@ class PandaModuleFinder(modulefinder.ModuleFinder): if name in self.badmodules: self._add_badmodule(name, caller) return + + if level <= 0 and caller and caller.__name__ in ignoreImports: + if name in ignoreImports[caller.__name__]: + return + try: self.import_hook(name, caller, level=level) except ImportError as msg: From d043df7d4e3cc4c10c07f89f9e52f32ba8cb7898 Mon Sep 17 00:00:00 2001 From: rdb Date: Mon, 18 Jan 2021 23:37:23 +0100 Subject: [PATCH 9/9] task: Add delay= argument to taskMgr.add() This has the same effect as doMethodLater, but slightly better describes what it does --- direct/src/task/Task.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/direct/src/task/Task.py b/direct/src/task/Task.py index bc7b9b6e6a..7be8e077d4 100644 --- a/direct/src/task/Task.py +++ b/direct/src/task/Task.py @@ -333,7 +333,7 @@ class TaskManager: def add(self, funcOrTask, name = None, sort = None, extraArgs = None, priority = None, uponDeath = None, appendTask = False, - taskChain = None, owner = None): + taskChain = None, owner = None, delay = None): """ Add a new task to the taskMgr. The task will begin executing immediately, or next frame if its sort value has already @@ -386,12 +386,17 @@ class TaskManager: is called when the task terminates. This is all the ownermeans. + delay: an optional amount of seconds to wait before starting + the task (equivalent to doMethodLater). + Returns: The new Task object that has been added, or the original Task object that was passed in. """ task = self.__setupTask(funcOrTask, name, priority, sort, extraArgs, taskChain, appendTask, owner, uponDeath) + if delay is not None: + task.setDelay(delay) self.mgr.add(task) return task