bam version 4.8; compress channels more conservatively

panda/src/chan/animChannelMatrixXfmTable.cxx

@@ -264,9 +264,9 @@ write_datagram(BamWriter *manager, Datagram &me) {
 
   } else {
     // Write out everything using lossy compression.
-
     FFTCompressor compressor;
     compressor.set_quality(compress_chan_quality);
+    compressor.set_use_error_threshold(true);
     compressor.write_header(me);
 
     // First, write out the scales and shears.
@@ -337,7 +337,7 @@ fillin(DatagramIterator &scan, BamReader *manager) {
     }
 
     FFTCompressor compressor;
-    compressor.read_header(scan);
+    compressor.read_header(scan, manager->get_file_minor_ver());
 
     int i;
     // First, read in the scales and shears.

panda/src/chan/animChannelScalarTable.cxx

@@ -21,12 +21,12 @@
 #include "animBundle.h"
 #include "config_chan.h"
 
-#include <indent.h>
+#include "indent.h"
-#include <datagram.h>
+#include "datagram.h"
-#include <datagramIterator.h>
+#include "datagramIterator.h"
-#include <bamReader.h>
+#include "bamReader.h"
-#include <bamWriter.h>
+#include "bamWriter.h"
-#include <fftCompressor.h>
+#include "fftCompressor.h"
 
 TypeHandle AnimChannelScalarTable::_type_handle;
 
@@ -225,6 +225,7 @@ write_datagram(BamWriter *manager, Datagram &me)
 
       FFTCompressor compressor;
       compressor.set_quality(compress_chan_quality);
+      compressor.set_use_error_threshold(true);
       compressor.write_header(me);
 
       compressor.write_reals(me, _table, _table.size());
@@ -298,7 +299,7 @@ fillin(DatagramIterator& scan, BamReader* manager)
     } else {
       // Continuous channels.
       FFTCompressor compressor;
-      compressor.read_header(scan);
+      compressor.read_header(scan, manager->get_file_minor_ver());
       compressor.read_reals(scan, temp_table.v());
     }
   }

panda/src/egg2pg/characterMaker.cxx

@@ -31,6 +31,7 @@
 #include "transformState.h"
 #include "eggSurface.h"
 #include "eggCurve.h"
+#include "modelNode.h"
 
 ////////////////////////////////////////////////////////////////////
 //     Function: CharacterMaker::Construtor
@@ -190,7 +191,9 @@ build_joint_hierarchy(EggNode *egg_node, PartGroup *part) {
       if (egg_group->get_dcs_type() != EggGroup::DC_none) {
         // If the joint requested an explicit DCS, create a node for
         // it.
-        joint->_geom_node = new PandaNode(egg_group->get_name());
+        PT(ModelNode) geom_node = new ModelNode(egg_group->get_name());
+        geom_node->set_preserve_transform(ModelNode::PT_local);
+        joint->_geom_node = geom_node.p();
       }
 
       part = joint;

panda/src/mathutil/config_mathutil.cxx

@@ -33,6 +33,7 @@ NotifyCategoryDef(mathutil, "");
 const double fft_offset = config_mathutil.GetDouble("fft-offset", 0.001);
 const double fft_factor = config_mathutil.GetDouble("fft-factor", 0.1);
 const double fft_exponent = config_mathutil.GetDouble("fft-exponent", 4);
+const double fft_error_threshold = config_mathutil.GetDouble("fft-error-threshold", 0.2);
 
 ConfigureFn(config_mathutil) {
   BoundingHexahedron::init_type();

panda/src/mathutil/config_mathutil.h

@@ -27,6 +27,7 @@ NotifyCategoryDecl(mathutil, EXPCL_PANDA, EXPTP_PANDA);
 extern const double fft_offset;
 extern const double fft_factor;
 extern const double fft_exponent;
+extern const double fft_error_threshold;
 
 #endif
 

panda/src/mathutil/fftCompressor.cxx

@@ -47,7 +47,9 @@ static RealPlans _real_decompress_plans;
 ////////////////////////////////////////////////////////////////////
 FFTCompressor::
 FFTCompressor() {
+  _bam_minor_version = 0;
   set_quality(-1);
+  _use_error_threshold = false;
   _transpose_quats = false;
 }
 
@@ -161,6 +163,34 @@ get_quality() const {
   return _quality;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: FFTCompressor::set_use_error_threshold
+//       Access: Public
+//  Description: Enables or disables the use of the error threshold
+//               measurement to put a cap on the amount of damage done
+//               by lossy compression.  When this is enabled, the
+//               potential results of the compression are analyzed
+//               before the data is written; if it is determined that
+//               the compression will damage a particular string of
+//               reals too much, that particular string of reals is
+//               written uncompressed.
+////////////////////////////////////////////////////////////////////
+void FFTCompressor::
+set_use_error_threshold(bool use_error_threshold) {
+  _use_error_threshold = use_error_threshold;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: FFTCompressor::get_use_error_threshold
+//       Access: Public
+//  Description: Returns whether the error threshold measurement is
+//               enabled.  See set_use_error_threshold().
+////////////////////////////////////////////////////////////////////
+bool FFTCompressor::
+get_use_error_threshold() const {
+  return _use_error_threshold;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: FFTCompressor::set_transpose_quats
 //       Access: Public
@@ -251,6 +281,31 @@ write_reals(Datagram &datagram, const float *array, int length) {
   rfftw_plan plan = get_real_compress_plan(length);
   rfftw_one(plan, data, half_complex);
 
+  bool reject_compression = false;
+
+  if (_use_error_threshold) {
+    // As a sanity check, decode the numbers again and see how far off
+    // we will be from the original string.
+    double error = get_error(data, half_complex, length);
+    if (error > fft_error_threshold) {
+      // No good: the compression is too damage.  Just write out
+      // lossless data.
+      reject_compression = true;
+    }
+  }
+
+  datagram.add_bool(reject_compression);
+  if (reject_compression) {
+    if (mathutil_cat.is_debug()) {
+      mathutil_cat.debug()
+        << "Writing stream of " << length << " numbers uncompressed.\n";
+    }
+    for (int i = 0; i < length; i++) {
+      datagram.add_float32(array[i]);
+    }
+    return;
+  }
+
   // Now encode the numbers, run-length encoded by size, so we only
   // write out the number of bits we need for each number.
 
@@ -463,7 +518,8 @@ write_hprs(Datagram &datagram, const LVecBase3f *array, int length) {
 //               false otherwise.
 ////////////////////////////////////////////////////////////////////
 bool FFTCompressor::
-read_header(DatagramIterator &di) {
+read_header(DatagramIterator &di, int bam_minor_version) {
+  _bam_minor_version = bam_minor_version;
   _quality = di.get_int8();
 
   if (mathutil_cat.is_debug()) {
@@ -533,6 +589,21 @@ read_reals(DatagramIterator &di, vector_float &array) {
 
   // Normal case: read in the FFT array, and convert it back to
   // (nearly) the original numbers.
+
+  // First, check the reject_compression flag.  If it's set, we
+  // decided to just write out the stream uncompressed.
+  bool reject_compression = false;
+  if (_bam_minor_version >= 8) {
+    reject_compression = di.get_bool();
+  }
+  if (reject_compression) {
+    array.reserve(array.size() + length);
+    for (int i = 0; i < length; i++) {
+      array.push_back(di.get_float32());
+    }
+    return true;
+  }
+
   vector_double half_complex;
   half_complex.reserve(length);
   int num_read = 0;
@@ -898,6 +969,72 @@ interpolate(double t, double a, double b) {
   return a + t * (b - a);
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: FFTCompressor::get_error
+//       Access: Private
+//  Description: Measures the error that would be incurred from
+//               compressing the string of reals.
+////////////////////////////////////////////////////////////////////
+double FFTCompressor::
+get_error(const double *data, const double *half_complex, int length) const {
+  double *truncated_half_complex = (double *)alloca(length * sizeof(double));
+  int i;
+  for (i = 0; i < length; i++) {
+    double scale_factor = get_scale_factor(i, length);
+    double num = cfloor(half_complex[i] / scale_factor + 0.5);
+    truncated_half_complex[i] = num * scale_factor;
+  }
+
+  double *new_data = (double *)alloca(length * sizeof(double));
+  rfftw_plan plan = get_real_decompress_plan(length);
+  rfftw_one(plan, &truncated_half_complex[0], new_data);
+
+  double scale = 1.0 / (double)length;
+  for (i = 0; i < length; i++) {
+    new_data[i] *= scale;
+  }
+
+  double last_value = data[0];
+  double last_new_value = new_data[0];
+
+  for (i = 0; i < length; i++) {
+    // First, we get the delta from each frame to the next.
+    double next_value = data[i];
+    double data_delta = data[i] - last_value;
+    last_value = next_value;
+
+    double next_new_value = new_data[i];
+    double data_new_delta = new_data[i] - last_value;
+    last_new_value = next_new_value;
+
+    // And we store the relative change in delta between our original
+    // values and our compressed values.
+    new_data[i] = data_new_delta - data_delta;
+  }
+
+  // Our error measurement is nothing more than the standard deviation
+  // of the relative change in delta, from above.  If this is large,
+  // the compressed values are moving substantially more erratically
+  // than the original values.
+
+  double sum = 0.0;
+  double sum2 = 0.0;
+  for (i = 0; i < length; i++) {
+    sum += new_data[i];
+    sum2 += new_data[i] * new_data[i];
+  }
+  double variance = (sum2 - (sum * sum) / length) / (length - 1);
+  if (variance < 0.0) {
+    // This can only happen due to tiny roundoff error.
+    return 0.0;
+  }
+
+  double std_deviation = sqrt(variance);
+
+  return std_deviation;
+}
+
+
 
 #ifdef HAVE_FFTW
 

panda/src/mathutil/fftCompressor.h

@@ -54,6 +54,9 @@ public:
   void set_quality(int quality);
   int get_quality() const;
 
+  void set_use_error_threshold(bool use_error_threshold);
+  bool get_use_error_threshold() const;
+
   void set_transpose_quats(bool flag);
   bool get_transpose_quats() const;
 
@@ -61,7 +64,7 @@ public:
   void write_reals(Datagram &datagram, const float *array, int length);
   void write_hprs(Datagram &datagram, const LVecBase3f *array, int length);
 
-  bool read_header(DatagramIterator &di);
+  bool read_header(DatagramIterator &di, int bam_minor_version);
   bool read_reals(DatagramIterator &di, vector_float &array);
   bool read_hprs(DatagramIterator &di, vector_LVecBase3f &array);
 
@@ -89,7 +92,11 @@ private:
   double get_scale_factor(int i, int length) const;
   static double interpolate(double t, double a, double b);
 
+  double get_error(const double *data, const double *half_complex, int length) const;
+
+  int _bam_minor_version;
   int _quality;
+  bool _use_error_threshold;
   double _fft_offset;
   double _fft_factor;
   double _fft_exponent;

panda/src/putil/bam.h

@@ -34,7 +34,7 @@ static const unsigned short _bam_major_ver = 4;
 // Bumped to major version 3 on 12/8/00 to change float64's to float32's.
 // Bumped to major version 4 on 4/10/02 to store new scene graph.
 
-static const unsigned short _bam_minor_ver = 7;
+static const unsigned short _bam_minor_ver = 8;
 // Bumped to minor version 1 on 4/10/03 to add CullFaceAttrib::reverse.
 // Bumped to minor version 2 on 4/12/03 to add num_components to texture.
 // Bumped to minor version 3 on 4/15/03 to add ImageBuffer::_alpha_file_channel
@@ -42,6 +42,7 @@ static const unsigned short _bam_minor_ver = 7;
 // Bumped to minor version 5 on 7/09/03 to add rawdata mode to texture.
 // Bumped to minor version 6 on 7/22/03 to add shear to scene graph and animation data.
 // Bumped to minor version 7 on 11/10/03 to add CollisionSolid::_effective_normal
+// Bumped to minor version 8 on 11/12/03 to add FFTCompressor::reject_compression
 
 
 #endif