From 39224c651ddb2428f201c1681f731e6e0a742e47 Mon Sep 17 00:00:00 2001 From: rdb Date: Sun, 25 Aug 2019 09:46:32 +0200 Subject: [PATCH 01/22] test_wheel: remove temp hack for pip 19.2.2, as 19.2.3 is released The hack was introduced in 22833686e3b09edfeb881767178c233efe393229 [skip ci] --- makepanda/test_wheel.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/makepanda/test_wheel.py b/makepanda/test_wheel.py index ff1fdcee9a..86fb8f4d2f 100755 --- a/makepanda/test_wheel.py +++ b/makepanda/test_wheel.py @@ -36,14 +36,6 @@ def test_wheel(wheel, verbose=False): shutil.rmtree(envdir) sys.exit(1) - # Temp hack to patch issue pypa/pip#6885 in pip 19.2.2 and Python 3.8. - if sys.platform == "win32" and "-cp38-cp38-" in wheel and os.path.isdir(os.path.join(envdir, "Lib", "site-packages", "pip-19.2.2.dist-info")): - pep425tags = os.path.join(envdir, "Lib", "site-packages", "pip", "_internal", "pep425tags.py") - if os.path.isfile(pep425tags): - data = open(pep425tags, "r").read() - data = data.replace(" m = 'm'\n", " m = ''\n") - open(pep425tags, "w").write(data) - # Install pytest into the environment, as well as our wheel. if subprocess.call([python, "-m", "pip", "install", "pytest", wheel]) != 0: shutil.rmtree(envdir) From 7b5f818ab2bbcd9cb412f236a1cfa31e682a3f00 Mon Sep 17 00:00:00 2001 From: rdb Date: Mon, 26 Aug 2019 10:07:58 +0200 Subject: [PATCH 02/22] tests: convert makepanda/test_imports.py to a unit test --- .travis.yml | 1 - makepanda/test_imports.py | 333 -------------------------------------- tests/test_imports.py | 328 +++++++++++++++++++++++++++++++++++++ 3 files changed, 328 insertions(+), 334 deletions(-) delete mode 100644 makepanda/test_imports.py create mode 100644 tests/test_imports.py diff --git a/.travis.yml b/.travis.yml index c3991fbeb1..ee9b09d37e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,6 @@ install: - $PYTHONV -m pip install pytest script: - $PYTHONV makepanda/makepanda.py --everything --git-commit $TRAVIS_COMMIT $FLAGS --threads 4 - - test -n "$SKIP_TESTS" || LD_LIBRARY_PATH=built/lib PYTHONPATH=built $PYTHONV makepanda/test_imports.py - test -n "$SKIP_TESTS" || LD_LIBRARY_PATH=built/lib PYTHONPATH=built $PYTHONV -m pytest -v tests notifications: irc: diff --git a/makepanda/test_imports.py b/makepanda/test_imports.py deleted file mode 100644 index dd7fb5ad23..0000000000 --- a/makepanda/test_imports.py +++ /dev/null @@ -1,333 +0,0 @@ -# This script imports Panda3D modules just to make sure that there are no -# missing imports. It is useful for a quick and dirty test to make sure -# that there are no obvious build issues. -import os, importlib - -# This will print out imports on the command line. -import direct.showbase.VerboseImport - - -import imp -import panda3d -dir = os.path.dirname(panda3d.__file__) - -extensions = set() -for suffix in imp.get_suffixes(): - extensions.add(suffix[0]) - -for basename in os.listdir(dir): - module = basename.split('.', 1)[0] - ext = basename[len(module):] - - if ext in extensions: - importlib.import_module('panda3d.%s' % (module)) - - -import direct.actor.Actor -import direct.actor.DistributedActor -import direct.cluster.ClusterClient -import direct.cluster.ClusterConfig -import direct.cluster.ClusterMsgs -import direct.cluster.ClusterServer -import direct.controls.BattleWalker -import direct.controls.ControlManager -import direct.controls.DevWalker -import direct.controls.GhostWalker -import direct.controls.GravityWalker -import direct.controls.InputState -import direct.controls.NonPhysicsWalker -import direct.controls.ObserverWalker -import direct.controls.PhysicsRoller -import direct.controls.PhysicsWalker -import direct.controls.SwimWalker -import direct.controls.TwoDWalker -import direct.directnotify.DirectNotify -import direct.directnotify.DirectNotifyGlobal -import direct.directnotify.Logger -import direct.directnotify.LoggerGlobal -import direct.directnotify.Notifier -import direct.directnotify.RotatingLog -import direct.directtools.DirectCameraControl -import direct.directtools.DirectGeometry -import direct.directtools.DirectGlobals -import direct.directtools.DirectGrid -import direct.directtools.DirectLights -import direct.directtools.DirectManipulation -import direct.directtools.DirectSelection -import direct.directtools.DirectUtil -import direct.directutil.DeltaProfiler -import direct.directutil.DistributedLargeBlobSender -import direct.directutil.DistributedLargeBlobSenderAI -import direct.directutil.LargeBlobSenderConsts -import direct.directutil.Mopath -import direct.directutil.Verify -import direct.directutil.WeightedChoice -import direct.dist.FreezeTool -import direct.distributed.AsyncRequest -import direct.distributed.CRCache -import direct.distributed.CRDataCache -import direct.distributed.CachedDOData -import direct.distributed.CartesianGridBase -import direct.distributed.ClientRepository -import direct.distributed.ClientRepositoryBase -import direct.distributed.ClockDelta -import direct.distributed.ConnectionRepository -import direct.distributed.DistributedCamera -import direct.distributed.DistributedCameraAI -import direct.distributed.DistributedCameraOV -import direct.distributed.DistributedCartesianGrid -import direct.distributed.DistributedCartesianGridAI -import direct.distributed.DistributedNode -import direct.distributed.DistributedNodeAI -import direct.distributed.DistributedNodeUD -import direct.distributed.DistributedObject -import direct.distributed.DistributedObjectAI -import direct.distributed.DistributedObjectBase -import direct.distributed.DistributedObjectGlobal -import direct.distributed.DistributedObjectGlobalAI -import direct.distributed.DistributedObjectGlobalUD -import direct.distributed.DistributedObjectOV -import direct.distributed.DistributedObjectUD -import direct.distributed.DistributedSmoothNodeAI -import direct.distributed.DistributedSmoothNodeBase -import direct.distributed.DoCollectionManager -import direct.distributed.DoHierarchy -import direct.distributed.DoInterestManager -import direct.distributed.GridChild -import direct.distributed.GridParent -import direct.distributed.InterestWatcher -import direct.distributed.MsgTypes -import direct.distributed.MsgTypesCMU -import direct.distributed.NetMessenger -import direct.distributed.ParentMgr -import direct.distributed.PyDatagram -import direct.distributed.PyDatagramIterator -import direct.distributed.RelatedObjectMgr -import direct.distributed.SampleObject -import direct.distributed.ServerRepository -import direct.distributed.StagedObject -import direct.distributed.TimeManager -import direct.distributed.TimeManagerAI -import direct.extensions_native.extension_native_helpers -import direct.filter.CommonFilters -import direct.filter.FilterManager -import direct.fsm.ClassicFSM -import direct.fsm.FSM -import direct.fsm.FourState -import direct.fsm.FourStateAI -import direct.fsm.SampleFSM -import direct.fsm.State -import direct.fsm.StateData -import direct.fsm.StatePush -import direct.gui.DirectButton -import direct.gui.DirectCheckBox -import direct.gui.DirectCheckButton -import direct.gui.DirectDialog -import direct.gui.DirectEntry -import direct.gui.DirectEntryScroll -import direct.gui.DirectFrame -import direct.gui.DirectGui -import direct.gui.DirectGuiBase -import direct.gui.DirectGuiGlobals -import direct.gui.DirectGuiTest -import direct.gui.DirectLabel -import direct.gui.DirectOptionMenu -import direct.gui.DirectRadioButton -import direct.gui.DirectScrollBar -import direct.gui.DirectScrolledFrame -import direct.gui.DirectScrolledList -import direct.gui.DirectSlider -import direct.gui.DirectWaitBar -import direct.gui.OnscreenGeom -import direct.gui.OnscreenImage -import direct.gui.OnscreenText -import direct.interval.ActorInterval -import direct.interval.AnimControlInterval -import direct.interval.FunctionInterval -import direct.interval.IndirectInterval -import direct.interval.Interval -import direct.interval.IntervalGlobal -import direct.interval.IntervalManager -import direct.interval.IntervalTest -import direct.interval.LerpBlendHelpers -import direct.interval.LerpInterval -import direct.interval.MetaInterval -import direct.interval.MopathInterval -import direct.interval.ParticleInterval -import direct.interval.ProjectileInterval -import direct.interval.ProjectileIntervalTest -import direct.interval.SoundInterval -import direct.interval.TestInterval -import direct.motiontrail.MotionTrail -import direct.p3d.AppRunner -import direct.p3d.DWBPackageInstaller -import direct.p3d.DeploymentTools -import direct.p3d.FileSpec -import direct.p3d.HostInfo -import direct.p3d.InstalledHostData -import direct.p3d.InstalledPackageData -import direct.p3d.JavaScript -import direct.p3d.PackageInfo -import direct.p3d.PackageInstaller -import direct.p3d.PackageMerger -import direct.p3d.Packager -import direct.p3d.PatchMaker -import direct.p3d.ScanDirectoryNode -import direct.p3d.SeqValue -import direct.particles.ForceGroup -import direct.particles.GlobalForceGroup -import direct.particles.ParticleEffect -import direct.particles.ParticleFloorTest -import direct.particles.ParticleManagerGlobal -import direct.particles.ParticleTest -import direct.particles.Particles -import direct.particles.SpriteParticleRendererExt -import direct.physics.FallTest -import direct.physics.RotationTest -import direct.showbase.AppRunnerGlobal -import direct.showbase.Audio3DManager -import direct.showbase.BufferViewer -import direct.showbase.BulletinBoard -import direct.showbase.BulletinBoardGlobal -import direct.showbase.BulletinBoardWatcher -import direct.showbase.ContainerLeakDetector -import direct.showbase.ContainerReport -import direct.showbase.CountedResource -import direct.showbase.DirectObject -import direct.showbase.DistancePhasedNode -import direct.showbase.EventGroup -import direct.showbase.EventManager -import direct.showbase.EventManagerGlobal -import direct.showbase.ExceptionVarDump -import direct.showbase.Factory -import direct.showbase.Finder -import direct.showbase.GarbageReport -import direct.showbase.GarbageReportScheduler -import direct.showbase.InputStateGlobal -import direct.showbase.Job -import direct.showbase.JobManager -import direct.showbase.JobManagerGlobal -import direct.showbase.LeakDetectors -import direct.showbase.Loader -import direct.showbase.Messenger -import direct.showbase.MessengerGlobal -import direct.showbase.MessengerLeakDetector -import direct.showbase.MirrorDemo -import direct.showbase.ObjectPool -import direct.showbase.ObjectReport -import direct.showbase.OnScreenDebug -import direct.showbase.PhasedObject -import direct.showbase.PhysicsManagerGlobal -import direct.showbase.Pool -import direct.showbase.ProfileSession -import direct.showbase.PythonUtil -import direct.showbase.RandomNumGen -import direct.showbase.ReferrerSearch -import direct.showbase.SfxPlayer -import direct.showbase.ShadowDemo -import direct.showbase.ShadowPlacer -import direct.showbase.ShowBase -import direct.showbase.TaskThreaded -import direct.showbase.ThreeUpShow -import direct.showbase.Transitions -import direct.showbase.VFSImporter -import direct.showbase.WxGlobal -import direct.showutil.BuildGeometry -import direct.showutil.Effects -import direct.showutil.Rope -import direct.showutil.TexMemWatcher -import direct.showutil.TexViewer -import direct.stdpy.file -import direct.stdpy.glob -#import direct.stdpy.pickle -import direct.stdpy.thread -import direct.stdpy.threading -import direct.stdpy.threading2 -import direct.task.FrameProfiler -import direct.task.MiniTask -import direct.task.Task -import direct.task.TaskManagerGlobal -import direct.task.TaskProfiler -import direct.task.TaskTester -import direct.task.Timer - -try: - import Pmw -except ImportError: - Pmw = None - print("Skipping Tk modules due to missing Pmw.") - -if Pmw: - import direct.showbase.TkGlobal - import direct.tkpanels.AnimPanel - import direct.tkpanels.DirectSessionPanel - import direct.tkpanels.FSMInspector - import direct.tkpanels.Inspector - import direct.tkpanels.MopathRecorder - import direct.tkpanels.NotifyPanel - import direct.tkpanels.ParticlePanel - import direct.tkpanels.Placer - import direct.tkpanels.TaskManagerPanel - import direct.tkwidgets.AppShell - import direct.tkwidgets.Dial - import direct.tkwidgets.EntryScale - import direct.tkwidgets.Floater - import direct.tkwidgets.MemoryExplorer - import direct.tkwidgets.ProgressBar - import direct.tkwidgets.SceneGraphExplorer - import direct.tkwidgets.Slider - import direct.tkwidgets.Tree - import direct.tkwidgets.Valuator - import direct.tkwidgets.VectorWidgets - import direct.tkwidgets.WidgetPropertiesDialog - -try: - import wx -except ImportError: - wx = None - print("Skipping wxPython and leveleditor modules due to missing wx.") - -if wx: - import direct.wxwidgets.ViewPort - import direct.wxwidgets.WxAppShell - import direct.wxwidgets.WxPandaShell - import direct.wxwidgets.WxPandaWindow - import direct.wxwidgets.WxSlider - - import direct.leveleditor.ActionMgr - import direct.leveleditor.AnimControlUI - import direct.leveleditor.AnimGlobals - import direct.leveleditor.AnimMgr - import direct.leveleditor.AnimMgrBase - import direct.leveleditor.CurveAnimUI - import direct.leveleditor.CurveEditor - import direct.leveleditor.FileMgr - import direct.leveleditor.GraphEditorUI - import direct.leveleditor.HotKeyUI - import direct.leveleditor.LayerEditorUI - import direct.leveleditor.LevelEditor - import direct.leveleditor.LevelEditorBase - import direct.leveleditor.LevelEditorStart - import direct.leveleditor.LevelEditorUI - import direct.leveleditor.LevelEditorUIBase - import direct.leveleditor.LevelLoader - import direct.leveleditor.LevelLoaderBase - import direct.leveleditor.MayaConverter - import direct.leveleditor.ObjectGlobals - import direct.leveleditor.ObjectHandler - import direct.leveleditor.ObjectMgr - import direct.leveleditor.ObjectMgrBase - import direct.leveleditor.ObjectPalette - import direct.leveleditor.ObjectPaletteBase - import direct.leveleditor.ObjectPaletteUI - import direct.leveleditor.ObjectPropertyUI - import direct.leveleditor.PaletteTreeCtrl - import direct.leveleditor.ProtoObjs - import direct.leveleditor.ProtoObjsUI - import direct.leveleditor.ProtoPalette - import direct.leveleditor.ProtoPaletteBase - import direct.leveleditor.ProtoPaletteUI - import direct.leveleditor.SceneGraphUI - import direct.leveleditor.SceneGraphUIBase - -print("Finished.") diff --git a/tests/test_imports.py b/tests/test_imports.py new file mode 100644 index 0000000000..7dd4f0deee --- /dev/null +++ b/tests/test_imports.py @@ -0,0 +1,328 @@ +# These tests import Panda3D modules just to make sure that there are no +# missing imports. It is useful for a quick and dirty test to make sure +# that there are no obvious build issues. +import pytest + +# This will print out imports on the command line. +#import direct.showbase.VerboseImport + + + +def test_imports_panda3d(): + import imp, importlib, os + import panda3d + dir = os.path.dirname(panda3d.__file__) + + extensions = set() + for suffix in imp.get_suffixes(): + extensions.add(suffix[0]) + + for basename in os.listdir(dir): + module = basename.split('.', 1)[0] + ext = basename[len(module):] + + if ext in extensions: + importlib.import_module('panda3d.%s' % (module)) + + +def test_imports_direct(): + import direct.actor.Actor + import direct.actor.DistributedActor + import direct.cluster.ClusterClient + import direct.cluster.ClusterConfig + import direct.cluster.ClusterMsgs + import direct.cluster.ClusterServer + import direct.controls.BattleWalker + import direct.controls.ControlManager + import direct.controls.DevWalker + import direct.controls.GhostWalker + import direct.controls.GravityWalker + import direct.controls.InputState + import direct.controls.NonPhysicsWalker + import direct.controls.ObserverWalker + import direct.controls.PhysicsRoller + import direct.controls.PhysicsWalker + import direct.controls.SwimWalker + import direct.controls.TwoDWalker + import direct.directnotify.DirectNotify + import direct.directnotify.DirectNotifyGlobal + import direct.directnotify.Logger + import direct.directnotify.LoggerGlobal + import direct.directnotify.Notifier + import direct.directnotify.RotatingLog + import direct.directtools.DirectCameraControl + import direct.directtools.DirectGeometry + import direct.directtools.DirectGlobals + import direct.directtools.DirectGrid + import direct.directtools.DirectLights + import direct.directtools.DirectManipulation + import direct.directtools.DirectSelection + import direct.directtools.DirectUtil + import direct.directutil.DeltaProfiler + import direct.directutil.DistributedLargeBlobSender + import direct.directutil.DistributedLargeBlobSenderAI + import direct.directutil.LargeBlobSenderConsts + import direct.directutil.Mopath + import direct.directutil.Verify + import direct.directutil.WeightedChoice + import direct.dist.FreezeTool + import direct.distributed.AsyncRequest + import direct.distributed.CRCache + import direct.distributed.CRDataCache + import direct.distributed.CachedDOData + import direct.distributed.CartesianGridBase + import direct.distributed.ClientRepository + import direct.distributed.ClientRepositoryBase + import direct.distributed.ClockDelta + import direct.distributed.ConnectionRepository + import direct.distributed.DistributedCamera + import direct.distributed.DistributedCameraAI + import direct.distributed.DistributedCameraOV + import direct.distributed.DistributedCartesianGrid + import direct.distributed.DistributedCartesianGridAI + import direct.distributed.DistributedNode + import direct.distributed.DistributedNodeAI + import direct.distributed.DistributedNodeUD + import direct.distributed.DistributedObject + import direct.distributed.DistributedObjectAI + import direct.distributed.DistributedObjectBase + import direct.distributed.DistributedObjectGlobal + import direct.distributed.DistributedObjectGlobalAI + import direct.distributed.DistributedObjectGlobalUD + import direct.distributed.DistributedObjectOV + import direct.distributed.DistributedObjectUD + import direct.distributed.DistributedSmoothNodeAI + import direct.distributed.DistributedSmoothNodeBase + import direct.distributed.DoCollectionManager + import direct.distributed.DoHierarchy + import direct.distributed.DoInterestManager + import direct.distributed.GridChild + import direct.distributed.GridParent + import direct.distributed.InterestWatcher + import direct.distributed.MsgTypes + import direct.distributed.MsgTypesCMU + import direct.distributed.NetMessenger + import direct.distributed.ParentMgr + import direct.distributed.PyDatagram + import direct.distributed.PyDatagramIterator + import direct.distributed.RelatedObjectMgr + import direct.distributed.SampleObject + import direct.distributed.ServerRepository + import direct.distributed.StagedObject + import direct.distributed.TimeManager + import direct.distributed.TimeManagerAI + import direct.extensions_native.extension_native_helpers + import direct.filter.CommonFilters + import direct.filter.FilterManager + import direct.fsm.ClassicFSM + import direct.fsm.FSM + import direct.fsm.FourState + import direct.fsm.FourStateAI + import direct.fsm.SampleFSM + import direct.fsm.State + import direct.fsm.StateData + import direct.fsm.StatePush + import direct.gui.DirectButton + import direct.gui.DirectCheckBox + import direct.gui.DirectCheckButton + import direct.gui.DirectDialog + import direct.gui.DirectEntry + import direct.gui.DirectEntryScroll + import direct.gui.DirectFrame + import direct.gui.DirectGui + import direct.gui.DirectGuiBase + import direct.gui.DirectGuiGlobals + import direct.gui.DirectGuiTest + import direct.gui.DirectLabel + import direct.gui.DirectOptionMenu + import direct.gui.DirectRadioButton + import direct.gui.DirectScrollBar + import direct.gui.DirectScrolledFrame + import direct.gui.DirectScrolledList + import direct.gui.DirectSlider + import direct.gui.DirectWaitBar + import direct.gui.OnscreenGeom + import direct.gui.OnscreenImage + import direct.gui.OnscreenText + import direct.interval.ActorInterval + import direct.interval.AnimControlInterval + import direct.interval.FunctionInterval + import direct.interval.IndirectInterval + import direct.interval.Interval + import direct.interval.IntervalGlobal + import direct.interval.IntervalManager + import direct.interval.IntervalTest + import direct.interval.LerpBlendHelpers + import direct.interval.LerpInterval + import direct.interval.MetaInterval + import direct.interval.MopathInterval + import direct.interval.ParticleInterval + import direct.interval.ProjectileInterval + import direct.interval.ProjectileIntervalTest + import direct.interval.SoundInterval + import direct.interval.TestInterval + import direct.motiontrail.MotionTrail + import direct.p3d.AppRunner + import direct.p3d.DWBPackageInstaller + import direct.p3d.DeploymentTools + import direct.p3d.FileSpec + import direct.p3d.HostInfo + import direct.p3d.InstalledHostData + import direct.p3d.InstalledPackageData + import direct.p3d.JavaScript + import direct.p3d.PackageInfo + import direct.p3d.PackageInstaller + import direct.p3d.PackageMerger + import direct.p3d.Packager + import direct.p3d.PatchMaker + import direct.p3d.ScanDirectoryNode + import direct.p3d.SeqValue + import direct.particles.ForceGroup + import direct.particles.GlobalForceGroup + import direct.particles.ParticleEffect + import direct.particles.ParticleFloorTest + import direct.particles.ParticleManagerGlobal + import direct.particles.ParticleTest + import direct.particles.Particles + import direct.particles.SpriteParticleRendererExt + import direct.physics.FallTest + import direct.physics.RotationTest + import direct.showbase.AppRunnerGlobal + import direct.showbase.Audio3DManager + import direct.showbase.BufferViewer + import direct.showbase.BulletinBoard + import direct.showbase.BulletinBoardGlobal + import direct.showbase.BulletinBoardWatcher + import direct.showbase.ContainerLeakDetector + import direct.showbase.ContainerReport + import direct.showbase.CountedResource + import direct.showbase.DirectObject + import direct.showbase.DistancePhasedNode + import direct.showbase.EventGroup + import direct.showbase.EventManager + import direct.showbase.EventManagerGlobal + import direct.showbase.ExceptionVarDump + import direct.showbase.Factory + import direct.showbase.Finder + import direct.showbase.GarbageReport + import direct.showbase.GarbageReportScheduler + import direct.showbase.InputStateGlobal + import direct.showbase.Job + import direct.showbase.JobManager + import direct.showbase.JobManagerGlobal + import direct.showbase.LeakDetectors + import direct.showbase.Loader + import direct.showbase.Messenger + import direct.showbase.MessengerGlobal + import direct.showbase.MessengerLeakDetector + import direct.showbase.MirrorDemo + import direct.showbase.ObjectPool + import direct.showbase.ObjectReport + import direct.showbase.OnScreenDebug + import direct.showbase.PhasedObject + import direct.showbase.PhysicsManagerGlobal + import direct.showbase.Pool + import direct.showbase.ProfileSession + import direct.showbase.PythonUtil + import direct.showbase.RandomNumGen + import direct.showbase.ReferrerSearch + import direct.showbase.SfxPlayer + import direct.showbase.ShadowDemo + import direct.showbase.ShadowPlacer + import direct.showbase.ShowBase + import direct.showbase.TaskThreaded + import direct.showbase.ThreeUpShow + import direct.showbase.Transitions + import direct.showbase.VFSImporter + import direct.showbase.WxGlobal + import direct.showutil.BuildGeometry + import direct.showutil.Effects + import direct.showutil.Rope + import direct.showutil.TexMemWatcher + import direct.showutil.TexViewer + import direct.stdpy.file + import direct.stdpy.glob + import direct.stdpy.pickle + import direct.stdpy.thread + import direct.stdpy.threading + import direct.stdpy.threading2 + import direct.task.FrameProfiler + import direct.task.MiniTask + import direct.task.Task + import direct.task.TaskManagerGlobal + import direct.task.TaskProfiler + import direct.task.TaskTester + import direct.task.Timer + + +def test_imports_tk(): + Pmw = pytest.importorskip('Pmw') + + import direct.showbase.TkGlobal + import direct.tkpanels.AnimPanel + import direct.tkpanels.DirectSessionPanel + import direct.tkpanels.FSMInspector + import direct.tkpanels.Inspector + import direct.tkpanels.MopathRecorder + import direct.tkpanels.NotifyPanel + import direct.tkpanels.ParticlePanel + import direct.tkpanels.Placer + import direct.tkpanels.TaskManagerPanel + import direct.tkwidgets.AppShell + import direct.tkwidgets.Dial + import direct.tkwidgets.EntryScale + import direct.tkwidgets.Floater + import direct.tkwidgets.MemoryExplorer + import direct.tkwidgets.ProgressBar + import direct.tkwidgets.SceneGraphExplorer + import direct.tkwidgets.Slider + import direct.tkwidgets.Tree + import direct.tkwidgets.Valuator + import direct.tkwidgets.VectorWidgets + import direct.tkwidgets.WidgetPropertiesDialog + + +def test_imports_wx(): + wx = pytest.importorskip('wx') + + import direct.wxwidgets.ViewPort + import direct.wxwidgets.WxAppShell + import direct.wxwidgets.WxPandaShell + import direct.wxwidgets.WxPandaWindow + import direct.wxwidgets.WxSlider + + import direct.leveleditor.ActionMgr + import direct.leveleditor.AnimControlUI + import direct.leveleditor.AnimGlobals + import direct.leveleditor.AnimMgr + import direct.leveleditor.AnimMgrBase + import direct.leveleditor.CurveAnimUI + import direct.leveleditor.CurveEditor + import direct.leveleditor.FileMgr + import direct.leveleditor.GraphEditorUI + import direct.leveleditor.HotKeyUI + import direct.leveleditor.LayerEditorUI + import direct.leveleditor.LevelEditor + import direct.leveleditor.LevelEditorBase + import direct.leveleditor.LevelEditorStart + import direct.leveleditor.LevelEditorUI + import direct.leveleditor.LevelEditorUIBase + import direct.leveleditor.LevelLoader + import direct.leveleditor.LevelLoaderBase + import direct.leveleditor.MayaConverter + import direct.leveleditor.ObjectGlobals + import direct.leveleditor.ObjectHandler + import direct.leveleditor.ObjectMgr + import direct.leveleditor.ObjectMgrBase + import direct.leveleditor.ObjectPalette + import direct.leveleditor.ObjectPaletteBase + import direct.leveleditor.ObjectPaletteUI + import direct.leveleditor.ObjectPropertyUI + import direct.leveleditor.PaletteTreeCtrl + import direct.leveleditor.ProtoObjs + import direct.leveleditor.ProtoObjsUI + import direct.leveleditor.ProtoPalette + import direct.leveleditor.ProtoPaletteBase + import direct.leveleditor.ProtoPaletteUI + import direct.leveleditor.SceneGraphUI + import direct.leveleditor.SceneGraphUIBase From 61a276f49a7696ee2794739fb966ae02bdead0a3 Mon Sep 17 00:00:00 2001 From: rdb Date: Wed, 28 Aug 2019 11:03:56 +0200 Subject: [PATCH 03/22] tests: add some ModifierButtons unit tests --- tests/putil/test_modifierbuttons.py | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/putil/test_modifierbuttons.py diff --git a/tests/putil/test_modifierbuttons.py b/tests/putil/test_modifierbuttons.py new file mode 100644 index 0000000000..289966d1df --- /dev/null +++ b/tests/putil/test_modifierbuttons.py @@ -0,0 +1,35 @@ +from panda3d.core import ModifierButtons + + +def test_modifierbuttons_empty(): + # Tests the initial state of a ModifierButtons object. + btns = ModifierButtons() + assert btns == ModifierButtons(btns) + assert btns != ModifierButtons() + assert btns.matches(ModifierButtons()) + assert not btns.is_down("alt") + assert not btns.is_any_down() + assert not btns.has_button("alt") + assert btns.get_prefix() == "" + assert btns.get_num_buttons() == 0 + assert len(btns.buttons) == 0 + + +def test_modifierbuttons_cow(): + # Tests the copy-on-write mechanism of the button list. + btns1 = ModifierButtons() + btns1.add_button("space") + + # Modifying original should not affect copy + btns2 = ModifierButtons(btns1) + assert tuple(btns2.buttons) == tuple(btns1.buttons) + btns1.add_button("enter") + assert tuple(btns1.buttons) == ("space", "enter") + assert tuple(btns2.buttons) == ("space",) + + # Modifying copy should not affect original + btns3 = ModifierButtons(btns2) + assert tuple(btns3.buttons) == tuple(btns2.buttons) + btns3.add_button("escape") + assert tuple(btns2.buttons) == ("space",) + assert tuple(btns3.buttons) == ("space", "escape") From 00376c9d0a4f5c309c8fc4a9b78af7cbcaa7c878 Mon Sep 17 00:00:00 2001 From: rdb Date: Wed, 28 Aug 2019 22:41:38 +0200 Subject: [PATCH 04/22] deploy-ng: fix some Python 3 issues in icon generation code See #718 --- direct/src/p3d/DeploymentTools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/direct/src/p3d/DeploymentTools.py b/direct/src/p3d/DeploymentTools.py index 1849556dcd..56ce034d84 100644 --- a/direct/src/p3d/DeploymentTools.py +++ b/direct/src/p3d/DeploymentTools.py @@ -367,7 +367,7 @@ class Icon: # XOR mask if bpp == 24: # Align rows to 4-byte boundary - rowalign = '\0' * (-(size * 3) & 3) + rowalign = b'\0' * (-(size * 3) & 3) for y in xrange(size): for x in xrange(size): r, g, b = image.getXel(x, size - y - 1) @@ -503,7 +503,7 @@ class Icon: if size > 256: continue elif size == 256: - ico.write('\0\0') + ico.write(b'\0\0') else: ico.write(struct.pack(' Date: Wed, 28 Aug 2019 22:43:30 +0200 Subject: [PATCH 05/22] pnmimage: add quantize feature to reduce number of colors in image --- panda/src/pnmimage/pnmImage.I | 168 ++++++++++++++++------------- panda/src/pnmimage/pnmImage.cxx | 131 ++++++++++++++++++++++ panda/src/pnmimage/pnmImage.h | 6 ++ panda/src/pnmimage/pnmimage_base.h | 16 +++ tests/pnmimage/test_pnmimage.py | 51 +++++++++ 5 files changed, 295 insertions(+), 77 deletions(-) diff --git a/panda/src/pnmimage/pnmImage.I b/panda/src/pnmimage/pnmImage.I index fe589d3f9b..a82e6d6eda 100644 --- a/panda/src/pnmimage/pnmImage.I +++ b/panda/src/pnmimage/pnmImage.I @@ -71,6 +71,57 @@ clamp_val(int input_value) const { return (xelval)std::min(std::max(0, input_value), (int)get_maxval()); } +/** + * A handy function to scale non-alpha values from [0..1] to + * [0..get_maxval()]. Do not use this for alpha values, see to_alpha_val. + */ +INLINE xel PNMImage:: +to_val(const LRGBColorf &value) const { + xel col; + switch (_xel_encoding) { + case XE_generic: + case XE_generic_alpha: + { + LRGBColorf scaled = value * get_maxval() + 0.5f; + col.r = clamp_val((int)scaled[0]); + col.g = clamp_val((int)scaled[1]); + col.b = clamp_val((int)scaled[2]); + } + break; + + case XE_generic_sRGB: + case XE_generic_sRGB_alpha: + col.r = clamp_val((int) + (encode_sRGB_float(value[0]) * get_maxval() + 0.5f)); + col.g = clamp_val((int) + (encode_sRGB_float(value[1]) * get_maxval() + 0.5f)); + col.b = clamp_val((int) + (encode_sRGB_float(value[2]) * get_maxval() + 0.5f)); + break; + + case XE_uchar_sRGB: + case XE_uchar_sRGB_alpha: + encode_sRGB_uchar(LColorf(value, 0.0f), col); + break; + + case XE_uchar_sRGB_sse2: + case XE_uchar_sRGB_alpha_sse2: + encode_sRGB_uchar_sse2(LColorf(value, 0.0f), col); + break; + + case XE_scRGB: + case XE_scRGB_alpha: + { + LRGBColorf scaled = value * 8192.f + 4096.5f; + col.r = std::min(std::max(0, (int)scaled[0]), 65535); + col.g = std::min(std::max(0, (int)scaled[1]), 65535); + col.b = std::min(std::max(0, (int)scaled[2]), 65535); + } + break; + } + return col; +} + /** * A handy function to scale non-alpha values from [0..1] to * [0..get_maxval()]. Do not use this for alpha values, see to_alpha_val. @@ -112,6 +163,44 @@ to_alpha_val(float input_value) const { return clamp_val((int)(input_value * get_maxval() + 0.5)); } +/** + * A handy function to scale non-alpha values from [0..get_maxval()] to + * [0..1]. Do not use this for alpha values, see from_alpha_val. + */ +INLINE LRGBColorf PNMImage:: +from_val(const xel &col) const { + switch (_xel_encoding) { + case XE_generic: + case XE_generic_alpha: + return LRGBColorf(col.r, col.g, col.b) * _inv_maxval; + + case XE_generic_sRGB: + case XE_generic_sRGB_alpha: + return LRGBColorf( + decode_sRGB_float(col.r * _inv_maxval), + decode_sRGB_float(col.g * _inv_maxval), + decode_sRGB_float(col.b * _inv_maxval)); + + case XE_uchar_sRGB: + case XE_uchar_sRGB_alpha: + case XE_uchar_sRGB_sse2: + case XE_uchar_sRGB_alpha_sse2: + return LRGBColorf( + decode_sRGB_float((unsigned char)col.r), + decode_sRGB_float((unsigned char)col.g), + decode_sRGB_float((unsigned char)col.b)); + + case XE_scRGB: + case XE_scRGB_alpha: + return LRGBColorf((int)col.r - 4096, + (int)col.g - 4096, + (int)col.b - 4096) * (1.f / 8192.f); + + default: + return LRGBColorf(0); + } +} + /** * A handy function to scale non-alpha values from [0..get_maxval()] to * [0..1]. Do not use this for alpha values, see from_alpha_val. @@ -479,39 +568,7 @@ set_alpha_val(int x, int y, xelval a) { INLINE LRGBColorf PNMImage:: get_xel(int x, int y) const { nassertr(x >= 0 && x < _x_size && y >= 0 && y < _y_size, LRGBColorf::zero()); - - const xel &col = row(y)[x]; - - switch (_xel_encoding) { - case XE_generic: - case XE_generic_alpha: - return LRGBColorf(col.r, col.g, col.b) * _inv_maxval; - - case XE_generic_sRGB: - case XE_generic_sRGB_alpha: - return LRGBColorf( - decode_sRGB_float(col.r * _inv_maxval), - decode_sRGB_float(col.g * _inv_maxval), - decode_sRGB_float(col.b * _inv_maxval)); - - case XE_uchar_sRGB: - case XE_uchar_sRGB_alpha: - case XE_uchar_sRGB_sse2: - case XE_uchar_sRGB_alpha_sse2: - return LRGBColorf( - decode_sRGB_float((unsigned char)col.r), - decode_sRGB_float((unsigned char)col.g), - decode_sRGB_float((unsigned char)col.b)); - - case XE_scRGB: - case XE_scRGB_alpha: - return LRGBColorf((int)col.r - 4096, - (int)col.g - 4096, - (int)col.b - 4096) * (1.f / 8192.f); - - default: - return LRGBColorf(0); - } + return from_val(row(y)[x]); } /** @@ -521,50 +578,7 @@ get_xel(int x, int y) const { INLINE void PNMImage:: set_xel(int x, int y, const LRGBColorf &value) { nassertv(x >= 0 && x < _x_size && y >= 0 && y < _y_size); - - xel &col = row(y)[x]; - - switch (_xel_encoding) { - case XE_generic: - case XE_generic_alpha: - { - LRGBColorf scaled = value * get_maxval() + 0.5f; - col.r = clamp_val((int)scaled[0]); - col.g = clamp_val((int)scaled[1]); - col.b = clamp_val((int)scaled[2]); - } - break; - - case XE_generic_sRGB: - case XE_generic_sRGB_alpha: - col.r = clamp_val((int) - (encode_sRGB_float(value[0]) * get_maxval() + 0.5f)); - col.g = clamp_val((int) - (encode_sRGB_float(value[1]) * get_maxval() + 0.5f)); - col.b = clamp_val((int) - (encode_sRGB_float(value[2]) * get_maxval() + 0.5f)); - break; - - case XE_uchar_sRGB: - case XE_uchar_sRGB_alpha: - encode_sRGB_uchar(LColorf(value, 0.0f), col); - break; - - case XE_uchar_sRGB_sse2: - case XE_uchar_sRGB_alpha_sse2: - encode_sRGB_uchar_sse2(LColorf(value, 0.0f), col); - break; - - case XE_scRGB: - case XE_scRGB_alpha: - { - LRGBColorf scaled = value * 8192.f + 4096.5f; - col.r = std::min(std::max(0, (int)scaled[0]), 65535); - col.g = std::min(std::max(0, (int)scaled[1]), 65535); - col.b = std::min(std::max(0, (int)scaled[2]), 65535); - } - break; - } + row(y)[x] = to_val(value); } /** diff --git a/panda/src/pnmimage/pnmImage.cxx b/panda/src/pnmimage/pnmImage.cxx index a26528f99a..a8deae0b6b 100644 --- a/panda/src/pnmimage/pnmImage.cxx +++ b/panda/src/pnmimage/pnmImage.cxx @@ -1928,6 +1928,51 @@ make_histogram(PNMImage::Histogram &histogram) { histogram.swap(pixels, hist_map); } +/** + * Reduces the number of unique colors in the image to (at most) the given + * count. Fewer colors than requested may be left in the image after this + * operation, but never more. + * + * At present, this is only supported on images without an alpha channel. + * + * @since 1.10.5 + */ +void PNMImage:: +quantize(size_t max_colors) { + nassertv(_array != nullptr); + nassertv(!has_alpha()); + size_t array_size = _x_size * _y_size; + + // Get all the unique colors in this image. + pmap color_map; + for (size_t i = 0; i < array_size; ++i) { + color_map[_array[i]]; + } + + size_t num_colors = color_map.size(); + if (num_colors <= max_colors) { + // We are already down to the requested number of colors. + return; + } + + // Collect all the colors into a contiguous array. + xel *colors = (xel *)alloca(num_colors * sizeof(xel)); + size_t i = 0; + for (pmap::const_iterator it = color_map.begin(); + it != color_map.end(); ++it) { + colors[i++] = it->first; + } + nassertv(i == num_colors); + + // Apply the median cut algorithm, which will give us a color map. + r_quantize(color_map, max_colors, colors, num_colors); + + // Replace all the existing colors with the corresponding bucket average. + for (size_t i = 0; i < array_size; ++i) { + _array[i] = color_map[_array[i]]; + } +} + /** * Fills the image with a grayscale perlin noise pattern based on the * indicated parameters. Uses set_xel to set the grayscale values. The sx @@ -2161,6 +2206,92 @@ setup_encoding() { } } +/** + * Recursive implementation of quantize() using the median cut algorithm. + */ +void PNMImage:: +r_quantize(pmap &color_map, size_t max_colors, + xel *colors, size_t num_colors) { + if (num_colors <= max_colors) { + // All points in this bucket can be preserved 1:1. + for (size_t i = 0; i < num_colors; ++i) { + const xel &col = colors[i]; + color_map[col] = col; + } + return; + } + else if (max_colors == 1) { + // We've reached the target. Calculate the average, in linear space. + LRGBColorf avg(0); + for (size_t i = 0; i < num_colors; ++i) { + avg += from_val(colors[i]); + } + avg *= 1.0f / num_colors; + xel avg_val = to_val(avg); + + // Map all colors in this bucket to the avg. + for (size_t i = 0; i < num_colors; ++i) { + color_map[colors[i]] = avg_val; + } + return; + } + else if (max_colors == 0) { + // Not sure how this happens, but we can't preserve any color here. + return; + } + + // Find the minimum/maximum RGB values. We should probably do this in + // linear space, but eh. + xelval min_r = _maxval; + xelval min_g = _maxval; + xelval min_b = _maxval; + xelval max_r = 0, max_g = 0, max_b = 0; + for (size_t i = 0; i < num_colors; ++i) { + const xel &col = colors[i]; + min_r = std::min(min_r, col.r); + max_r = std::max(max_r, col.r); + min_g = std::min(min_g, col.g); + max_g = std::max(max_g, col.g); + min_b = std::min(min_b, col.b); + max_b = std::max(max_b, col.b); + } + + int diff_r = max_r - min_r; + int diff_g = max_g - min_g; + int diff_b = max_b - min_b; + + auto sort_by_red = [](const xel &c1, const xel &c2) { + return c1.r < c2.r; + }; + auto sort_by_green = [](const xel &c1, const xel &c2) { + return c1.g < c2.g; + }; + auto sort_by_blue = [](const xel &c1, const xel &c2) { + return c1.b < c2.b; + }; + + // Sort by the component with the most variation. + if (diff_g >= diff_r) { + if (diff_g >= diff_b) { + std::sort(colors, colors + num_colors, sort_by_green); + } else { + std::sort(colors, colors + num_colors, sort_by_blue); + } + } else if (diff_r >= diff_b) { + std::sort(colors, colors + num_colors, sort_by_red); + } else { + std::sort(colors, colors + num_colors, sort_by_blue); + } + + // Subdivide the sorted colors into two buckets, and recurse. + size_t max_colors_1 = max_colors / 2; + size_t max_colors_2 = max_colors - max_colors_1; + size_t num_colors_1 = num_colors / 2; + size_t num_colors_2 = num_colors - num_colors_1; + r_quantize(color_map, max_colors_1, colors, num_colors_1); + r_quantize(color_map, max_colors_2, colors + num_colors_1, num_colors_2); +} + /** * Recursively fills in the minimum distance measured from a certain set of * points into the gray channel. diff --git a/panda/src/pnmimage/pnmImage.h b/panda/src/pnmimage/pnmImage.h index 494acb8029..da1e2d0553 100644 --- a/panda/src/pnmimage/pnmImage.h +++ b/panda/src/pnmimage/pnmImage.h @@ -68,8 +68,10 @@ PUBLISHED: INLINE ~PNMImage(); INLINE xelval clamp_val(int input_value) const; + INLINE xel to_val(const LRGBColorf &input_value) const; INLINE xelval to_val(float input_value) const; INLINE xelval to_alpha_val(float input_value) const; + INLINE LRGBColorf from_val(const xel &input_value) const; INLINE float from_val(xelval input_value) const; INLINE float from_alpha_val(xelval input_value) const; @@ -254,6 +256,7 @@ PUBLISHED: int xborder = 0, int yborder = 0); void make_histogram(Histogram &hist); + void quantize(size_t max_colors); BLOCKING void perlin_noise_fill(float sx, float sy, int table_size = 256, unsigned long seed = 0); void perlin_noise_fill(StackedPerlinNoise2 &perlin); @@ -346,6 +349,9 @@ private: void setup_rc(); void setup_encoding(); + void r_quantize(pmap &color_map, size_t max_colors, + xel *colors, size_t num_colors); + PUBLISHED: PNMImage operator ~() const; diff --git a/panda/src/pnmimage/pnmimage_base.h b/panda/src/pnmimage/pnmimage_base.h index ca1f356c48..aa5474e82a 100644 --- a/panda/src/pnmimage/pnmimage_base.h +++ b/panda/src/pnmimage/pnmimage_base.h @@ -59,6 +59,22 @@ PUBLISHED: void operator *= (const double mult) { r *= mult; g *= mult; b *= mult; } + bool operator == (const pixel &other) { + return r == other.r && g == other.g && r == other.r; + } + bool operator != (const pixel &other) { + return r != other.r || g != other.g || r != other.r; + } + bool operator < (const pixel &other) const { + if (r != other.r) { + return r < other.r; + } + if (g != other.g) { + return g < other.g; + } + return b < other.b; + } + #ifdef HAVE_PYTHON static int size() { return 3; } void output(std::ostream &out) { diff --git a/tests/pnmimage/test_pnmimage.py b/tests/pnmimage/test_pnmimage.py index 0826e4f0b8..40d8a88683 100644 --- a/tests/pnmimage/test_pnmimage.py +++ b/tests/pnmimage/test_pnmimage.py @@ -1,4 +1,5 @@ from panda3d.core import PNMImage, PNMImageHeader +from random import randint def test_pixelspec_ctor(): @@ -19,3 +20,53 @@ def test_pixelspec_coerce(): img = PNMImage(1, 1, 4) img.set_pixel(0, 0, (1, 2, 3, 4)) assert img.get_pixel(0, 0) == (1, 2, 3, 4) + + +def test_pnmimage_to_val(): + img = PNMImage(1, 1) + assert img.to_val(-0.5) == 0 + assert img.to_val(0.0) == 0 + assert img.to_val(0.5) == 128 + assert img.to_val(1.0) == 255 + assert img.to_val(2.0) == 255 + + +def test_pnmimage_from_val(): + img = PNMImage(1, 1) + assert img.from_val(0) == 0.0 + assert img.to_val(img.from_val(128)) == 128 + assert img.from_val(255) == 1.0 + + +def test_pnmimage_quantize(): + img = PNMImage(32, 32, 3) + + for x in range(32): + for y in range(32): + img.set_xel_val(x, y, randint(0, 100), randint(50, 100), randint(0, 1)) + + hist = PNMImage.Histogram() + img.make_histogram(hist) + num_colors = hist.get_num_pixels() + assert num_colors > 100 + + img2 = PNMImage(img) + img2.quantize(100) + hist = PNMImage.Histogram() + img2.make_histogram(hist) + assert hist.get_num_pixels() <= 100 + + # Make sure that this is reasonably close + max_dist = 0 + for x in range(32): + for y in range(32): + diff = img.get_xel(x, y) - img2.get_xel(x, y) + max_dist = max(max_dist, diff.length_squared()) + + # Also make sure that they are not out of range of the original + col = img2.get_xel_val(x, y) + assert col.r <= 100 + assert col.g >= 50 and col.g <= 100 + assert col.b in (0, 1) + + assert max_dist < 0.1 ** 2 From 28d9a33db44ead53b8a9ab06b5745531b1554f7e Mon Sep 17 00:00:00 2001 From: rdb Date: Wed, 28 Aug 2019 22:56:11 +0200 Subject: [PATCH 06/22] deploy-ng: fix an exception when all icon images could not be read --- direct/src/p3d/DeploymentTools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/direct/src/p3d/DeploymentTools.py b/direct/src/p3d/DeploymentTools.py index 56ce034d84..b67d4d69be 100644 --- a/direct/src/p3d/DeploymentTools.py +++ b/direct/src/p3d/DeploymentTools.py @@ -344,6 +344,7 @@ class Icon: if required_size * 2 in sizes: from_size = required_size * 2 else: + from_size = 0 for from_size in sizes: if from_size > required_size: break From d843c5c198194d6f0c03032122f0a58c88d1c4de Mon Sep 17 00:00:00 2001 From: rdb Date: Wed, 28 Aug 2019 22:51:28 +0200 Subject: [PATCH 07/22] deploy-ng: use median cut algorithm when palettizing icon This prevents infrequently occurring colors from being washed out. See #718 --- direct/src/p3d/DeploymentTools.py | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/direct/src/p3d/DeploymentTools.py b/direct/src/p3d/DeploymentTools.py index b67d4d69be..ebc20bfda9 100644 --- a/direct/src/p3d/DeploymentTools.py +++ b/direct/src/p3d/DeploymentTools.py @@ -384,35 +384,14 @@ class Icon: elif bpp == 8: # We'll have to generate a palette of 256 colors. hist = PNMImage.Histogram() - if image.hasAlpha(): - # Make a copy without alpha channel. - image2 = PNMImage(image) + image2 = PNMImage(image) + if image2.hasAlpha(): image2.premultiplyAlpha() image2.removeAlpha() - else: - image2 = image + image2.quantize(256) image2.make_histogram(hist) colors = list(hist.get_pixels()) - if len(colors) > 256: - # Palette too large; remove infrequent colors. - colors.sort(key=hist.get_count, reverse=True) - - # Find the closest color on the palette matching each color - # that didn't fit. This is certainly not the best palette - # generation code, but it'll do for now. - closest_indices = [] - for color in colors[256:]: - closest_index = 0 - closest_diff = 1025 - for i, closest_color in enumerate(colors[:256]): - diff = abs(color.get_red() - closest_color.get_red()) \ - + abs(color.get_green() - closest_color.get_green()) \ - + abs(color.get_blue() - closest_color.get_blue()) - if diff < closest_diff: - closest_index = i - closest_diff = diff - assert closest_diff < 100 - closest_indices.append(closest_index) + assert len(colors) <= 256 # Write the palette. i = 0 From 4d254177ba91d80def5582f4541c3c22b1545c20 Mon Sep 17 00:00:00 2001 From: rdb Date: Sun, 1 Sep 2019 21:32:52 +0200 Subject: [PATCH 08/22] general: Fix missing notifyCategoryProxy.h includes --- pandatool/src/assimp/config_assimp.h | 1 + pandatool/src/ptloader/config_ptloader.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pandatool/src/assimp/config_assimp.h b/pandatool/src/assimp/config_assimp.h index 898497eeb2..e17e2caae6 100644 --- a/pandatool/src/assimp/config_assimp.h +++ b/pandatool/src/assimp/config_assimp.h @@ -15,6 +15,7 @@ #define CONFIG_ASSIMP_H #include "pandatoolbase.h" +#include "notifyCategoryProxy.h" #include "configVariableBool.h" #include "configVariableDouble.h" #include "dconfig.h" diff --git a/pandatool/src/ptloader/config_ptloader.h b/pandatool/src/ptloader/config_ptloader.h index b195083b2b..fc324dc939 100644 --- a/pandatool/src/ptloader/config_ptloader.h +++ b/pandatool/src/ptloader/config_ptloader.h @@ -15,7 +15,7 @@ #define CONFIG_PTLOADER_H #include "pandatoolbase.h" - +#include "notifyCategoryProxy.h" #include "dconfig.h" #include "distanceUnit.h" #include "configVariableEnum.h" From 191a5b09e9a286dc430a516db8091a327ef1892c Mon Sep 17 00:00:00 2001 From: rdb Date: Sun, 1 Sep 2019 21:42:40 +0200 Subject: [PATCH 09/22] bin2c: fix compilation warning --- pandatool/src/miscprogs/binToC.cxx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pandatool/src/miscprogs/binToC.cxx b/pandatool/src/miscprogs/binToC.cxx index 6eff47c9d1..ae9fbc4090 100644 --- a/pandatool/src/miscprogs/binToC.cxx +++ b/pandatool/src/miscprogs/binToC.cxx @@ -99,8 +99,7 @@ run() { out << std::hex << std::setfill('0'); int count = 0; int col = 0; - unsigned int ch; - ch = in.get(); + int ch = in.get(); while (!in.fail() && ch != EOF) { if (col == 0) { out << "\n "; @@ -110,7 +109,7 @@ run() { } else { out << ", "; } - out << "0x" << std::setw(2) << ch; + out << "0x" << std::setw(2) << (unsigned int)ch; col++; count++; ch = in.get(); From 57ff1a5441c9fa9f74dd2b4dc96e5b9c52feb7d7 Mon Sep 17 00:00:00 2001 From: rdb Date: Tue, 3 Sep 2019 12:15:02 +0200 Subject: [PATCH 10/22] interrogate: remove vestigial code causing assert for C bindings Fixes #722 --- dtool/src/interrogate/interrogateBuilder.cxx | 32 +++----------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/dtool/src/interrogate/interrogateBuilder.cxx b/dtool/src/interrogate/interrogateBuilder.cxx index 7473d085d3..4735157d1d 100644 --- a/dtool/src/interrogate/interrogateBuilder.cxx +++ b/dtool/src/interrogate/interrogateBuilder.cxx @@ -2038,33 +2038,11 @@ get_make_property(CPPMakeProperty *make_property, CPPStructType *struct_type, CP iproperty._length_function = length_function; } - if (make_property->_type == CPPMakeProperty::T_normal) { - if (getter != nullptr) { - iproperty._flags |= InterrogateElement::F_has_getter; - iproperty._getter = get_function(getter, "", struct_type, - struct_type->get_scope(), 0); - nassertr(iproperty._getter, 0); - } - } else { - // We could have a mixed sequence/mapping property, so synthesize a - // getitem function. We don't really care what's in here; we just use - // this to store the remaps. - if (!iproperty.has_getter()) { - iproperty._flags |= InterrogateElement::F_has_getter; - iproperty._getter = InterrogateDatabase::get_ptr()->get_next_index(); - InterrogateFunction *ifunction = new InterrogateFunction; - ifunction->_instances = new InterrogateFunction::Instances; - InterrogateDatabase::get_ptr()->add_function(iproperty._getter, ifunction); - } - - // Add our getter to the generated getitem function. - string signature = TypeManager::get_function_signature(getter); - InterrogateFunction &ifunction = - InterrogateDatabase::get_ptr()->update_function(iproperty._getter); - if (ifunction._instances == nullptr) { - ifunction._instances = new InterrogateFunction::Instances; - } - ifunction._instances->insert(InterrogateFunction::Instances::value_type(signature, getter)); + if (getter != nullptr) { + iproperty._flags |= InterrogateElement::F_has_getter; + iproperty._getter = get_function(getter, "", struct_type, + struct_type->get_scope(), 0); + nassertr(iproperty._getter, 0); } if (hasser != nullptr) { From 2e03eb405b00a61543a36e2b82a0200be9ae742c Mon Sep 17 00:00:00 2001 From: rdb Date: Tue, 3 Sep 2019 22:29:54 +0200 Subject: [PATCH 11/22] deploy-ng: handle .pz and .gz models correctly --- direct/src/dist/commands.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/direct/src/dist/commands.py b/direct/src/dist/commands.py index 8a865f98dc..f8160ed546 100644 --- a/direct/src/dist/commands.py +++ b/direct/src/dist/commands.py @@ -66,6 +66,8 @@ def _parse_dict(input): def egg2bam(_build_cmd, srcpath, dstpath): + if dstpath.endswith('.gz') or dstpath.endswith('.pz'): + dstpath = dstpath[:-3] dstpath = dstpath + '.bam' try: subprocess.check_call([ @@ -895,6 +897,9 @@ class build_apps(setuptools.Command): os.makedirs(dst_dir) ext = os.path.splitext(src)[1] + # If the file ends with .gz/.pz, we strip this off. + if ext in ('.gz', '.pz'): + ext = os.path.splitext(src[:-3])[1] if not ext: ext = os.path.basename(src) From 79c71a5d978c19eb75c977fca2f1b05937e84ef7 Mon Sep 17 00:00:00 2001 From: rdb Date: Tue, 3 Sep 2019 22:31:06 +0200 Subject: [PATCH 12/22] pgraph: fix missing includes of *Collection classes in nodePath.h --- panda/src/pgraph/nodePath.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/panda/src/pgraph/nodePath.h b/panda/src/pgraph/nodePath.h index 4f0849fac4..eaec935a29 100644 --- a/panda/src/pgraph/nodePath.h +++ b/panda/src/pgraph/nodePath.h @@ -41,6 +41,8 @@ #include "pta_LVecBase2.h" #include "stl_compares.h" #include "shaderInput.h" +#include "internalNameCollection.h" +#include "materialCollection.h" #include "textureCollection.h" #include "textureStageCollection.h" @@ -49,13 +51,9 @@ class FindApproxPath; class FindApproxLevelEntry; class Light; class PolylightNode; -class InternalNameCollection; class Texture; class TextureStage; -class TextureCollection; -class TextureStageCollection; class Material; -class MaterialCollection; class Fog; class GlobPattern; class PreparedGraphicsObjects; @@ -1054,6 +1052,8 @@ private: INLINE std::ostream &operator << (std::ostream &out, const NodePath &node_path); +#include "nodePathCollection.h" + #include "nodePath.I" #endif From 323f74cb55861afbd936a69d279f364d79e28a02 Mon Sep 17 00:00:00 2001 From: rdb Date: Wed, 4 Sep 2019 10:19:01 +0200 Subject: [PATCH 13/22] event: squelch unprotected NewFrame event spam warning --- panda/src/event/eventQueue.cxx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/panda/src/event/eventQueue.cxx b/panda/src/event/eventQueue.cxx index 69c025734c..426bf161de 100644 --- a/panda/src/event/eventQueue.cxx +++ b/panda/src/event/eventQueue.cxx @@ -46,11 +46,13 @@ queue_event(CPT_Event event) { LightMutexHolder holder(_lock); _queue.push_back(event); - if (event_cat.is_spam() || event_cat.is_debug()) { + if (event_cat.is_debug()) { if (event->get_name() == "NewFrame") { // Don't bother us with this particularly spammy event. - event_cat.spam() - << "Throwing event " << *event << "\n"; + if (event_cat.is_spam()) { + event_cat.spam() + << "Throwing event " << *event << "\n"; + } } else { event_cat.debug() << "Throwing event " << *event << "\n"; From ba1023efa9817ee2dbc03b06ff24873e299447c2 Mon Sep 17 00:00:00 2001 From: rdb Date: Wed, 4 Sep 2019 12:26:43 +0200 Subject: [PATCH 14/22] stdpy: clarify comment to indicate os.PathLike is supported in open() --- direct/src/stdpy/file.py | 1 + 1 file changed, 1 insertion(+) diff --git a/direct/src/stdpy/file.py b/direct/src/stdpy/file.py index 222739612a..394fde2436 100644 --- a/direct/src/stdpy/file.py +++ b/direct/src/stdpy/file.py @@ -83,6 +83,7 @@ def open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, elif isinstance(file, strType): filename = core.Filename.fromOsSpecific(file) else: + # It's either a Filename object or an os.PathLike. # If a Filename is given, make a writable copy anyway. filename = core.Filename(file) From 7f7cbd1c25a9297b81c802f47b0f3d643c5b4556 Mon Sep 17 00:00:00 2001 From: rdb Date: Wed, 4 Sep 2019 12:40:58 +0200 Subject: [PATCH 15/22] tests: add Filename unit tests for fspath protocol in particular --- tests/dtoolutil/test_filename.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 tests/dtoolutil/test_filename.py diff --git a/tests/dtoolutil/test_filename.py b/tests/dtoolutil/test_filename.py new file mode 100644 index 0000000000..d24d9d52d7 --- /dev/null +++ b/tests/dtoolutil/test_filename.py @@ -0,0 +1,24 @@ +from panda3d.core import Filename +import sys, os +import pytest + + +@pytest.mark.skipif(sys.version_info < (3, 6), reason="Requires Python 3.6") +def test_filename_fspath(): + fn = Filename.from_os_specific(__file__) + assert os.fspath(fn) == fn.to_os_specific_w() + + +@pytest.mark.skipif(sys.version_info < (3, 6), reason="Requires Python 3.6") +def test_filename_open(): + fn = Filename.from_os_specific(__file__) + open(fn, 'rb') + + +@pytest.mark.skipif(sys.version_info < (3, 4), reason="Requires Python 3.4") +def test_filename_ctor_pathlib(): + pathlib = pytest.importorskip('pathlib') + + path = pathlib.Path(__file__) + fn = Filename(path) + assert fn.to_os_specific_w() == str(path) From b4fd82b8120f5b4a5018fc8d0e399708eb1a460f Mon Sep 17 00:00:00 2001 From: rdb Date: Fri, 6 Sep 2019 09:46:57 +0200 Subject: [PATCH 16/22] load_dso: we can expect LoadLibraryExW to be available --- dtool/src/dtoolutil/load_dso.cxx | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/dtool/src/dtoolutil/load_dso.cxx b/dtool/src/dtoolutil/load_dso.cxx index 0c7ae24ebe..fcd0c9fffe 100644 --- a/dtool/src/dtoolutil/load_dso.cxx +++ b/dtool/src/dtoolutil/load_dso.cxx @@ -50,19 +50,7 @@ load_dso(const DSearchPath &path, const Filename &filename) { return nullptr; } std::wstring os_specific_w = abspath.to_os_specific_w(); - - // Try using LoadLibraryEx, if possible. - typedef HMODULE (WINAPI *tLoadLibraryEx)(LPCWSTR, HANDLE, DWORD); - tLoadLibraryEx pLoadLibraryEx; - HINSTANCE hLib = LoadLibrary("kernel32.dll"); - if (hLib) { - pLoadLibraryEx = (tLoadLibraryEx)GetProcAddress(hLib, "LoadLibraryExW"); - if (pLoadLibraryEx) { - return pLoadLibraryEx(os_specific_w.c_str(), nullptr, LOAD_WITH_ALTERED_SEARCH_PATH); - } - } - - return LoadLibraryW(os_specific_w.c_str()); + return LoadLibraryExW(os_specific_w.c_str(), nullptr, LOAD_WITH_ALTERED_SEARCH_PATH); } bool From d1d244dfd241ed52045e9e7594b7ef5521feae2f Mon Sep 17 00:00:00 2001 From: rdb Date: Fri, 6 Sep 2019 09:48:11 +0200 Subject: [PATCH 17/22] pipeline: per-frame output message should be spam, not debug --- panda/src/pipeline/pipeline.cxx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/panda/src/pipeline/pipeline.cxx b/panda/src/pipeline/pipeline.cxx index a3633226bc..641b2c6e2c 100644 --- a/panda/src/pipeline/pipeline.cxx +++ b/panda/src/pipeline/pipeline.cxx @@ -86,8 +86,8 @@ Pipeline:: void Pipeline:: cycle() { #ifdef THREADED_PIPELINE - if (pipeline_cat.is_debug()) { - pipeline_cat.debug() + if (pipeline_cat.is_spam()) { + pipeline_cat.spam() << "Beginning the pipeline cycle\n"; } From 55d5fe30e67864d44908f1837af45530696239ab Mon Sep 17 00:00:00 2001 From: rdb Date: Fri, 6 Sep 2019 10:27:57 +0200 Subject: [PATCH 18/22] bullet: release GIL during expensive do_physics call Fixes #726 --- panda/src/bullet/bulletWorld.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/panda/src/bullet/bulletWorld.h b/panda/src/bullet/bulletWorld.h index bc032f1934..3218f92742 100644 --- a/panda/src/bullet/bulletWorld.h +++ b/panda/src/bullet/bulletWorld.h @@ -58,7 +58,7 @@ PUBLISHED: void set_gravity(PN_stdfloat gx, PN_stdfloat gy, PN_stdfloat gz); const LVector3 get_gravity() const; - int do_physics(PN_stdfloat dt, int max_substeps=1, PN_stdfloat stepsize=1.0f/60.0f); + BLOCKING int do_physics(PN_stdfloat dt, int max_substeps=1, PN_stdfloat stepsize=1.0f/60.0f); BulletSoftBodyWorldInfo get_world_info(); From aef81aceab51dee73b94dd48a1a30190b6d432d4 Mon Sep 17 00:00:00 2001 From: rdb Date: Fri, 6 Sep 2019 11:19:21 +0200 Subject: [PATCH 19/22] downloadertools: remove apply_patch and build_patch These seem to be unused. Let me know if this is not the case. --- makepanda/makepanda.py | 10 -- makepanda/makepanda.vcproj | 2 - panda/src/downloadertools/apply_patch.cxx | 46 -------- panda/src/downloadertools/build_patch.cxx | 125 ---------------------- 4 files changed, 183 deletions(-) delete mode 100644 panda/src/downloadertools/apply_patch.cxx delete mode 100644 panda/src/downloadertools/build_patch.cxx diff --git a/makepanda/makepanda.py b/makepanda/makepanda.py index 0eaf1cf8da..fe01ebd690 100755 --- a/makepanda/makepanda.py +++ b/makepanda/makepanda.py @@ -4605,16 +4605,6 @@ if PkgSkip("OPENAL") == 0 and not RUNTIME: if (PkgSkip("OPENSSL")==0 and not RTDIST and not RUNTIME and PkgSkip("DEPLOYTOOLS")==0): OPTS=['DIR:panda/src/downloadertools', 'OPENSSL', 'ZLIB', 'ADVAPI', 'WINSOCK2', 'WINSHELL', 'WINGDI', 'WINUSER'] - TargetAdd('apply_patch_apply_patch.obj', opts=OPTS, input='apply_patch.cxx') - TargetAdd('apply_patch.exe', input=['apply_patch_apply_patch.obj']) - TargetAdd('apply_patch.exe', input=COMMON_PANDA_LIBS) - TargetAdd('apply_patch.exe', opts=OPTS) - - TargetAdd('build_patch_build_patch.obj', opts=OPTS, input='build_patch.cxx') - TargetAdd('build_patch.exe', input=['build_patch_build_patch.obj']) - TargetAdd('build_patch.exe', input=COMMON_PANDA_LIBS) - TargetAdd('build_patch.exe', opts=OPTS) - if not PkgSkip("ZLIB"): TargetAdd('check_adler_check_adler.obj', opts=OPTS, input='check_adler.cxx') TargetAdd('check_adler.exe', input=['check_adler_check_adler.obj']) diff --git a/makepanda/makepanda.vcproj b/makepanda/makepanda.vcproj index c2db8fa662..31944e48e6 100644 --- a/makepanda/makepanda.vcproj +++ b/makepanda/makepanda.vcproj @@ -2087,8 +2087,6 @@ - - diff --git a/panda/src/downloadertools/apply_patch.cxx b/panda/src/downloadertools/apply_patch.cxx deleted file mode 100644 index 502d67c89b..0000000000 --- a/panda/src/downloadertools/apply_patch.cxx +++ /dev/null @@ -1,46 +0,0 @@ -/** - * PANDA 3D SOFTWARE - * Copyright (c) Carnegie Mellon University. All rights reserved. - * - * All use of this software is subject to the terms of the revised BSD - * license. You should have received a copy of this license along - * with this source code in a file named "LICENSE." - * - * @file apply_patch.cxx - */ - -#include "pandabase.h" -#include "panda_getopt.h" -#include "preprocess_argv.h" -#include "patchfile.h" -#include "filename.h" - -using std::cerr; -using std::endl; - -int -main(int argc, char **argv) { - preprocess_argv(argc, argv); - - if (argc < 3) { - cerr << "Usage: apply_patch " << endl; - cerr << "Will overwrite old_file" << endl; - return 1; - } - - Filename patch = argv[1]; - patch.set_binary(); - - Filename file = argv[2]; - file.set_binary(); - - Patchfile pfile; - - cerr << "Applying patch file " << patch << " to " << file << endl; - if (pfile.apply(patch, file) == false) { - cerr << "apply patch failed" << endl; - return 1; - } - - return 0; -} diff --git a/panda/src/downloadertools/build_patch.cxx b/panda/src/downloadertools/build_patch.cxx deleted file mode 100644 index 5c705bb38d..0000000000 --- a/panda/src/downloadertools/build_patch.cxx +++ /dev/null @@ -1,125 +0,0 @@ -/** - * PANDA 3D SOFTWARE - * Copyright (c) Carnegie Mellon University. All rights reserved. - * - * All use of this software is subject to the terms of the revised BSD - * license. You should have received a copy of this license along - * with this source code in a file named "LICENSE." - * - * @file build_patch.cxx - */ - -#include "pandabase.h" -#include "panda_getopt.h" -#include "preprocess_argv.h" -#include "patchfile.h" -#include "filename.h" - -using std::cerr; -using std::endl; - -void -usage() { - cerr << "Usage: build_patch [opts] " << endl; -} - -void -help() { - usage(); - cerr << "\n" - "This program generates a patch file that describes the differences\n" - "between any two source files. The patch file can later be used to\n" - "construct , given . Arbitrary file types, including\n" - "binary files, are supported.\n\n" - - "The patching algorithm can get very slow for very large files. As an\n" - "optimization, if the input files are both Panda Multifiles, the patcher\n" - "will by default patch them on a per-subfile basis, which has the potential\n" - "to be much faster. The -c option will forbid this and force the patcher\n" - "to work on the full file.\n\n" - - "Options:\n\n" - - " -o output_name\n" - " Specify the filename of the patch file to generate.\n\n" - - " -c\n" - " Always generate patches against the complete file, even if the\n" - " input files appear to be multifiles.\n\n" - - " -f footprint_length\n" - " Specify the footprint length for the patching algorithm.\n\n"; -} - -int -main(int argc, char **argv) { - Filename patch_file; - bool complete_file = false; - int footprint_length = 0; - - // extern char *optarg; - extern int optind; - static const char *optflags = "o:cf:h"; - preprocess_argv(argc, argv); - int flag = getopt(argc, argv, optflags); - Filename rel_path; - while (flag != EOF) { - switch (flag) { - case 'o': - patch_file = optarg; - break; - - case 'c': - complete_file = true; - break; - - case 'f': - footprint_length = atoi(optarg); - break; - - case 'h': - help(); - return 1; - case '?': - usage(); - return 1; - default: - cerr << "Unhandled switch: " << flag << endl; - break; - } - flag = getopt(argc, argv, optflags); - } - argc -= (optind - 1); - argv += (optind - 1); - - if (argc < 3) { - usage(); - return 1; - } - - Filename src_file = Filename::from_os_specific(argv[1]); - src_file.set_binary(); - - Filename dest_file = Filename::from_os_specific(argv[2]); - dest_file.set_binary(); - - if (patch_file.empty()) { - patch_file = dest_file.get_fullpath() + ".pch"; - } - Patchfile pfile; - - pfile.set_allow_multifile(!complete_file); - if (footprint_length != 0) { - cerr << "Footprint length is " << footprint_length << "\n"; - pfile.set_footprint_length(footprint_length); - } - - cerr << "Building patch file to convert " << src_file << " to " - << dest_file << endl; - if (pfile.build(src_file, dest_file, patch_file) == false) { - cerr << "build patch failed" << endl; - return 1; - } - - return 0; -} From 8b8796e9df6a302b7ec076b51e326f7f78c17560 Mon Sep 17 00:00:00 2001 From: rdb Date: Fri, 6 Sep 2019 11:21:15 +0200 Subject: [PATCH 20/22] express: remove libtar dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We haven't built with this for many years, and apparently nobody has missed it, so… good riddance? --- panda/src/express/patchfile.cxx | 229 -------------------------------- panda/src/express/patchfile.h | 29 ---- 2 files changed, 258 deletions(-) diff --git a/panda/src/express/patchfile.cxx b/panda/src/express/patchfile.cxx index ea6bb0e47b..bd5c4d9b8f 100644 --- a/panda/src/express/patchfile.cxx +++ b/panda/src/express/patchfile.cxx @@ -26,15 +26,6 @@ #include // for strstr -#ifdef HAVE_TAR -#include -#include // for O_RDONLY -#endif // HAVE_TAR - -#ifdef HAVE_TAR -std::istream *Patchfile::_tar_istream = nullptr; -#endif // HAVE_TAR - using std::endl; using std::ios; using std::istream; @@ -1160,202 +1151,6 @@ compute_mf_patches(ostream &write_stream, return true; } -#ifdef HAVE_TAR -/** - * Uses libtar to extract the location within the tar file of each of the - * subfiles. Returns true if the tar file is read successfully, false if - * there is an error (e.g. it is not a tar file). - */ -bool Patchfile:: -read_tar(TarDef &tar, istream &stream) { - TAR *tfile; - tartype_t tt; - tt.openfunc = tar_openfunc; - tt.closefunc = tar_closefunc; - tt.readfunc = tar_readfunc; - tt.writefunc = tar_writefunc; - - stream.seekg(0, ios::beg); - nassertr(_tar_istream == nullptr, false); - _tar_istream = &stream; - if (tar_open(&tfile, (char *)"dummy", &tt, O_RDONLY, 0, 0) != 0) { - _tar_istream = nullptr; - return false; - } - - // Walk through the tar file, noting the current file position as we reach - // each subfile. Use this information to infer the start and end of each - // subfile within the stream. - - streampos last_pos = 0; - int flag = th_read(tfile); - while (flag == 0) { - TarSubfile subfile; - subfile._name = th_get_pathname(tfile); - subfile._header_start = last_pos; - subfile._data_start = stream.tellg(); - subfile._data_end = subfile._data_start + (streampos)th_get_size(tfile); - tar_skip_regfile(tfile); - subfile._end = stream.tellg(); - tar.push_back(subfile); - - last_pos = subfile._end; - flag = th_read(tfile); - } - - // Create one more "subfile" for the bytes at the tail of the file. This - // subfile has no name. - TarSubfile subfile; - subfile._header_start = last_pos; - stream.clear(); - stream.seekg(0, ios::end); - subfile._data_start = stream.tellg(); - subfile._data_end = subfile._data_start; - subfile._end = subfile._data_start; - tar.push_back(subfile); - - tar_close(tfile); - _tar_istream = nullptr; - return (flag == 1); -} -#endif // HAVE_TAR - -#ifdef HAVE_TAR -/** - * Computes patches for the files, knowing that they are both tar files. This - * is similar to compute_mf_patches(). - * - * The tar indexes should have been built up by a previous call to read_tar(). - */ -bool Patchfile:: -compute_tar_patches(ostream &write_stream, - uint32_t offset_orig, uint32_t offset_new, - istream &stream_orig, istream &stream_new, - TarDef &tar_orig, TarDef &tar_new) { - - // Sort the orig list by filename, so we can quickly look up files from the - // new list. - tar_orig.sort(); - - // However, it is important to keep the new list in its original, on-disk - // order. - - // Walk through each subfile in the new tar file. If a particular subfile - // exists in both source files, we compute the patches for the subfile; for - // a new subfile, we trivially add it. If a subfile has been removed, we - // simply don't add it (we'll never even notice this case). - - IStreamWrapper stream_origw(stream_orig); - IStreamWrapper stream_neww(stream_new); - - TarDef::const_iterator ni; - streampos last_pos = 0; - for (ni = tar_new.begin(); ni != tar_new.end(); ++ni) { - const TarSubfile &sf_new =(*ni); - nassertr(sf_new._header_start == last_pos, false); - - TarDef::const_iterator oi = tar_orig.find(sf_new); - - if (oi == tar_orig.end()) { - // This is a newly-added subfile. Add it the hard way. - express_cat.info() - << "Adding subfile " << sf_new._name << "\n"; - - streampos new_start = sf_new._header_start; - size_t new_size = sf_new._end - sf_new._header_start; - char *buffer_new = (char *)PANDA_MALLOC_ARRAY(new_size); - stream_new.seekg(new_start, ios::beg); - stream_new.read(buffer_new, new_size); - cache_add_and_copy(write_stream, new_size, buffer_new, 0, 0); - PANDA_FREE_ARRAY(buffer_new); - - } else { - // This subfile exists in both the original and the new files. Patch - // it. - const TarSubfile &sf_orig =(*oi); - - // We patch the header and data of the file separately, so we can - // accurately detect nested multifiles. The extra data at the end of - // the file (possibly introduced by a tar file's blocking) is the - // footer, which is also patched separately. - if (!patch_subfile(write_stream, offset_orig, offset_new, "", - stream_origw, sf_orig._header_start, sf_orig._data_start, - stream_neww, sf_new._header_start, sf_new._data_start)) { - return false; - } - - if (!patch_subfile(write_stream, offset_orig, offset_new, sf_new._name, - stream_origw, sf_orig._data_start, sf_orig._data_end, - stream_neww, sf_new._data_start, sf_new._data_end)) { - return false; - } - - if (!patch_subfile(write_stream, offset_orig, offset_new, "", - stream_origw, sf_orig._data_end, sf_orig._end, - stream_neww, sf_new._data_end, sf_new._end)) { - return false; - } - } - - last_pos = sf_new._end; - } - - return true; -} -#endif // HAVE_TAR - -#ifdef HAVE_TAR -/** - * A callback function to redirect libtar to read from our istream instead of - * using low-level Unix I/O. - */ -int Patchfile:: -tar_openfunc(const char *, int, ...) { - // Since we don't actually open a file--the stream is already open--we do - // nothing here. - return 0; -} -#endif // HAVE_TAR - -#ifdef HAVE_TAR -/** - * A callback function to redirect libtar to read from our istream instead of - * using low-level Unix I/O. - */ -int Patchfile:: -tar_closefunc(int) { - // Since we don't actually open a file, no need to close it either. - return 0; -} -#endif // HAVE_TAR - -#ifdef HAVE_TAR -/** - * A callback function to redirect libtar to read from our istream instead of - * using low-level Unix I/O. - */ -ssize_t Patchfile:: -tar_readfunc(int, void *buffer, size_t nbytes) { - nassertr(_tar_istream != nullptr, 0); - _tar_istream->read((char *)buffer, nbytes); - return (ssize_t)_tar_istream->gcount(); -} -#endif // HAVE_TAR - -#ifdef HAVE_TAR -/** - * A callback function to redirect libtar to read from our istream instead of - * using low-level Unix I/O. - */ -ssize_t Patchfile:: -tar_writefunc(int, const void *, size_t) { - // Since we use libtar only for reading, it is an error if this method gets - // called. - nassertr(false, -1); - return -1; -} -#endif // HAVE_TAR - /** * * This implementation uses the "greedy differencing algorithm" described in @@ -1440,10 +1235,6 @@ do_compute_patches(const Filename &file_orig, const Filename &file_new, // Check whether our input files are Panda multifiles or tar files. bool is_multifile = false; -#ifdef HAVE_TAR - bool is_tarfile = false; - TarDef tar_orig, tar_new; -#endif // HAVE_TAR if (_allow_multifile) { if (strstr(file_orig.get_basename().c_str(), ".mf") != nullptr || @@ -1465,15 +1256,6 @@ do_compute_patches(const Filename &file_orig, const Filename &file_new, } PANDA_FREE_ARRAY(buffer); } -#ifdef HAVE_TAR - if (strstr(file_orig.get_basename().c_str(), ".tar") != nullptr || - strstr(file_new.get_basename().c_str(), ".tar") != nullptr) { - if (read_tar(tar_orig, stream_orig) && - read_tar(tar_new, stream_new)) { - is_tarfile = true; - } - } -#endif // HAVE_TAR } if (is_multifile) { @@ -1485,17 +1267,6 @@ do_compute_patches(const Filename &file_orig, const Filename &file_new, stream_orig, stream_new)) { return false; } -#ifdef HAVE_TAR - } else if (is_tarfile) { - if (express_cat.is_debug()) { - express_cat.debug() - << file_orig.get_basename() << " appears to be a tar file.\n"; - } - if (!compute_tar_patches(write_stream, offset_orig, offset_new, - stream_orig, stream_new, tar_orig, tar_new)) { - return false; - } -#endif // HAVE_TAR } else { if (express_cat.is_debug()) { express_cat.debug() diff --git a/panda/src/express/patchfile.h b/panda/src/express/patchfile.h index d052ef8386..852bd5b857 100644 --- a/panda/src/express/patchfile.h +++ b/panda/src/express/patchfile.h @@ -108,35 +108,6 @@ private: bool compute_mf_patches(std::ostream &write_stream, uint32_t offset_orig, uint32_t offset_new, std::istream &stream_orig, std::istream &stream_new); -#ifdef HAVE_TAR - class TarSubfile { - public: - inline bool operator < (const TarSubfile &other) const { - return _name < other._name; - } - std::string _name; - std::streampos _header_start; - std::streampos _data_start; - std::streampos _data_end; - std::streampos _end; - }; - typedef ov_set TarDef; - - bool read_tar(TarDef &tar, std::istream &stream); - bool compute_tar_patches(std::ostream &write_stream, - uint32_t offset_orig, uint32_t offset_new, - std::istream &stream_orig, std::istream &stream_new, - TarDef &tar_orig, TarDef &tar_new); - - // Because this is static, we can only call read_tar() one at a time--no - // threads, please. - static std::istream *_tar_istream; - - static int tar_openfunc(const char *filename, int oflags, ...); - static int tar_closefunc(int fd); - static ssize_t tar_readfunc(int fd, void *buffer, size_t nbytes); - static ssize_t tar_writefunc(int fd, const void *buffer, size_t nbytes); -#endif // HAVE_TAR bool do_compute_patches(const Filename &file_orig, const Filename &file_new, std::ostream &write_stream, From 13e91135e509497816ee2e180210ab9105ebc054 Mon Sep 17 00:00:00 2001 From: rdb Date: Fri, 6 Sep 2019 11:46:09 +0200 Subject: [PATCH 21/22] downloadertools: remove show_ddb --- makepanda/makepanda.py | 5 ---- makepanda/makepanda.vcproj | 1 - panda/src/downloadertools/show_ddb.cxx | 32 -------------------------- 3 files changed, 38 deletions(-) delete mode 100644 panda/src/downloadertools/show_ddb.cxx diff --git a/makepanda/makepanda.py b/makepanda/makepanda.py index fe01ebd690..436b64a9c6 100755 --- a/makepanda/makepanda.py +++ b/makepanda/makepanda.py @@ -4631,11 +4631,6 @@ if (PkgSkip("OPENSSL")==0 and not RTDIST and not RUNTIME and PkgSkip("DEPLOYTOOL TargetAdd('pencrypt.exe', input=COMMON_PANDA_LIBS) TargetAdd('pencrypt.exe', opts=OPTS) - TargetAdd('show_ddb_show_ddb.obj', opts=OPTS, input='show_ddb.cxx') - TargetAdd('show_ddb.exe', input=['show_ddb_show_ddb.obj']) - TargetAdd('show_ddb.exe', input=COMMON_PANDA_LIBS) - TargetAdd('show_ddb.exe', opts=OPTS) - # # DIRECTORY: panda/src/downloadertools/ # diff --git a/makepanda/makepanda.vcproj b/makepanda/makepanda.vcproj index 31944e48e6..f8825312b0 100644 --- a/makepanda/makepanda.vcproj +++ b/makepanda/makepanda.vcproj @@ -2079,7 +2079,6 @@ - diff --git a/panda/src/downloadertools/show_ddb.cxx b/panda/src/downloadertools/show_ddb.cxx deleted file mode 100644 index 1b1a4abce4..0000000000 --- a/panda/src/downloadertools/show_ddb.cxx +++ /dev/null @@ -1,32 +0,0 @@ -/** - * PANDA 3D SOFTWARE - * Copyright (c) Carnegie Mellon University. All rights reserved. - * - * All use of this software is subject to the terms of the revised BSD - * license. You should have received a copy of this license along - * with this source code in a file named "LICENSE." - * - * @file show_ddb.cxx - * @author drose - * @date 2002-11-02 - */ - -#include "pandabase.h" -#include "downloadDb.h" -#include "filename.h" - -int -main(int argc, char *argv[]) { - if (argc != 3) { - std::cerr << "Usage: show_ddb server.ddb client.ddb\n"; - return 1; - } - - Filename server_ddb = Filename::from_os_specific(argv[1]); - Filename client_ddb = Filename::from_os_specific(argv[2]); - - DownloadDb db(server_ddb, client_ddb); - db.write(std::cout); - - return 0; -} From 2e198fd2efee9229f1412fef165e4bd7bd7298d5 Mon Sep 17 00:00:00 2001 From: rdb Date: Fri, 6 Sep 2019 11:55:17 +0200 Subject: [PATCH 22/22] downloadertools: remove check_adler, check_md5, check_crc Tools to check hashes are readily available from thirdparty sources; there is no good reason why a 3D engine should provide them. --- makepanda/makepanda.py | 18 +-- makepanda/makepanda.vcproj | 3 - panda/src/downloadertools/check_adler.cxx | 26 ---- panda/src/downloadertools/check_crc.cxx | 26 ---- panda/src/downloadertools/check_md5.cxx | 155 ---------------------- 5 files changed, 1 insertion(+), 227 deletions(-) delete mode 100644 panda/src/downloadertools/check_adler.cxx delete mode 100644 panda/src/downloadertools/check_crc.cxx delete mode 100644 panda/src/downloadertools/check_md5.cxx diff --git a/makepanda/makepanda.py b/makepanda/makepanda.py index 436b64a9c6..c1f1056a1b 100755 --- a/makepanda/makepanda.py +++ b/makepanda/makepanda.py @@ -4603,23 +4603,7 @@ if PkgSkip("OPENAL") == 0 and not RUNTIME: # if (PkgSkip("OPENSSL")==0 and not RTDIST and not RUNTIME and PkgSkip("DEPLOYTOOLS")==0): - OPTS=['DIR:panda/src/downloadertools', 'OPENSSL', 'ZLIB', 'ADVAPI', 'WINSOCK2', 'WINSHELL', 'WINGDI', 'WINUSER'] - - if not PkgSkip("ZLIB"): - TargetAdd('check_adler_check_adler.obj', opts=OPTS, input='check_adler.cxx') - TargetAdd('check_adler.exe', input=['check_adler_check_adler.obj']) - TargetAdd('check_adler.exe', input=COMMON_PANDA_LIBS) - TargetAdd('check_adler.exe', opts=OPTS) - - TargetAdd('check_crc_check_crc.obj', opts=OPTS, input='check_crc.cxx') - TargetAdd('check_crc.exe', input=['check_crc_check_crc.obj']) - TargetAdd('check_crc.exe', input=COMMON_PANDA_LIBS) - TargetAdd('check_crc.exe', opts=OPTS) - - TargetAdd('check_md5_check_md5.obj', opts=OPTS, input='check_md5.cxx') - TargetAdd('check_md5.exe', input=['check_md5_check_md5.obj']) - TargetAdd('check_md5.exe', input=COMMON_PANDA_LIBS) - TargetAdd('check_md5.exe', opts=OPTS) + OPTS=['DIR:panda/src/downloadertools', 'OPENSSL', 'ADVAPI', 'WINSOCK2', 'WINSHELL', 'WINGDI', 'WINUSER'] TargetAdd('pdecrypt_pdecrypt.obj', opts=OPTS, input='pdecrypt.cxx') TargetAdd('pdecrypt.exe', input=['pdecrypt_pdecrypt.obj']) diff --git a/makepanda/makepanda.vcproj b/makepanda/makepanda.vcproj index f8825312b0..1cbb36aba6 100644 --- a/makepanda/makepanda.vcproj +++ b/makepanda/makepanda.vcproj @@ -2080,12 +2080,9 @@ - - - diff --git a/panda/src/downloadertools/check_adler.cxx b/panda/src/downloadertools/check_adler.cxx deleted file mode 100644 index dbea94f8b7..0000000000 --- a/panda/src/downloadertools/check_adler.cxx +++ /dev/null @@ -1,26 +0,0 @@ -/** - * PANDA 3D SOFTWARE - * Copyright (c) Carnegie Mellon University. All rights reserved. - * - * All use of this software is subject to the terms of the revised BSD - * license. You should have received a copy of this license along - * with this source code in a file named "LICENSE." - * - * @file check_adler.cxx - */ - -#include "download_utils.h" - -int -main(int argc, char *argv[]) { - if (argc < 2) { - std::cerr << "Usage: check_adler " << std::endl; - return 1; - } - - Filename source_file = argv[1]; - - std::cout << check_adler(source_file); - - return 0; -} diff --git a/panda/src/downloadertools/check_crc.cxx b/panda/src/downloadertools/check_crc.cxx deleted file mode 100644 index e98de20b57..0000000000 --- a/panda/src/downloadertools/check_crc.cxx +++ /dev/null @@ -1,26 +0,0 @@ -/** - * PANDA 3D SOFTWARE - * Copyright (c) Carnegie Mellon University. All rights reserved. - * - * All use of this software is subject to the terms of the revised BSD - * license. You should have received a copy of this license along - * with this source code in a file named "LICENSE." - * - * @file check_crc.cxx - */ - -#include "download_utils.h" - -int -main(int argc, char *argv[]) { - if (argc < 2) { - std::cerr << "Usage: check_crc " << std::endl; - return 1; - } - - Filename source_file = argv[1]; - - std::cout << check_crc(source_file); - - return 0; -} diff --git a/panda/src/downloadertools/check_md5.cxx b/panda/src/downloadertools/check_md5.cxx deleted file mode 100644 index 2cc3ce508e..0000000000 --- a/panda/src/downloadertools/check_md5.cxx +++ /dev/null @@ -1,155 +0,0 @@ -/** - * PANDA 3D SOFTWARE - * Copyright (c) Carnegie Mellon University. All rights reserved. - * - * All use of this software is subject to the terms of the revised BSD - * license. You should have received a copy of this license along - * with this source code in a file named "LICENSE." - * - * @file check_md5.cxx - */ - -#include "pandabase.h" -#include "hashVal.h" -#include "filename.h" -#include "panda_getopt.h" -#include "preprocess_argv.h" - -using std::cerr; -using std::cout; - -bool output_decimal = false; -bool suppress_filename = false; -pofstream binary_output; - -void -usage() { - cerr << - "\n" - "Usage:\n\n" - "check_md5 [-q] [-d] [-b filename] [-i \"input string\"] [file1 file2 ...]\n" - "check_md5 -h\n\n"; -} - -void -help() { - usage(); - cerr << - "This program outputs the MD5 hash of one or more files (or of a string\n" - "passed on the command line with -i).\n\n" - - "An MD5 hash is a 128-bit value. The output is presented as a 32-digit\n" - "hexadecimal string by default, but with -d, it is presented as four\n" - "big-endian unsigned 32-bit decimal integers. Normally the filename\n" - "of each file is printed along with the hash; -q suppresses this.\n\n" - - "To write the 16 bytes (per input file) of the output directly to a\n" - "binary file, use -b with the name of the file to receive the output.\n"; -} - -void -output_hash(const std::string &filename, const HashVal &hash) { - if (!suppress_filename && !filename.empty()) { - cout << filename << " "; - } - if (output_decimal) { - hash.output_dec(cout); - } else { - hash.output_hex(cout); - } - cout << "\n"; - - // Also output to the binary_output file if it is open. No sweat if it's - // not. - hash.output_binary(binary_output); -} - - -int -main(int argc, char **argv) { - extern char *optarg; - extern int optind; - const char *optstr = "i:db:qh"; - - bool got_input_string = false; - std::string input_string; - Filename binary_output_filename; - - preprocess_argv(argc, argv); - int flag = getopt(argc, argv, optstr); - - while (flag != EOF) { - switch (flag) { - case 'i': - got_input_string = true; - input_string = optarg; - break; - - case 'd': - output_decimal = true; - break; - - case 'b': - binary_output_filename = Filename::binary_filename(std::string(optarg)); - break; - - case 'q': - suppress_filename = true; - break; - - case 'h': - help(); - exit(1); - - default: - exit(1); - } - flag = getopt(argc, argv, optstr); - } - - argc -= (optind-1); - argv += (optind-1); - - if (argc < 2 && !got_input_string) { - usage(); - exit(1); - } - - if (!binary_output_filename.empty()) { - if (!binary_output_filename.open_write(binary_output)) { - cerr << "Unable to open " << binary_output_filename << ".\n"; - exit(1); - } - } - - if (got_input_string) { - HashVal hash; - hash.hash_string(input_string); - output_hash("", hash); - } - - bool okflag = true; - - for (int i = 1; i < argc; i++) { - Filename source_file = Filename::from_os_specific(argv[i]); - - if (!source_file.exists()) { - cerr << source_file << " not found!\n"; - okflag = false; - } else { - HashVal hash; - if (!hash.hash_file(source_file)) { - cerr << "Unable to read " << source_file << "\n"; - okflag = false; - } else { - output_hash(source_file.get_basename(), hash); - } - } - } - - if (!okflag) { - exit(1); - } - - return 0; -}