From fadbcd91e5f77019d9ec3e346247e3c8ded46f87 Mon Sep 17 00:00:00 2001 From: Sam Edwards Date: Sat, 31 Mar 2018 05:22:04 -0600 Subject: [PATCH 01/17] ffmpeg: Bump supported minimums lower for libav 9.20 We need to support this because Ubuntu 14.04 ships with this version, and has no "backports" option to bring in a newer version (without use of a PPA or compiling it yourself). We can consider raising the minimums again once Trusty is EOL. --- panda/src/ffmpeg/config_ffmpeg.cxx | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/panda/src/ffmpeg/config_ffmpeg.cxx b/panda/src/ffmpeg/config_ffmpeg.cxx index be788eb2f8..850cf62937 100644 --- a/panda/src/ffmpeg/config_ffmpeg.cxx +++ b/panda/src/ffmpeg/config_ffmpeg.cxx @@ -30,16 +30,19 @@ extern "C" { #error Buildsystem error: BUILDING_FFMPEG not defined #endif -#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(54, 86, 100) - #error Minimum supported version of libavcodec is 54.86.100. +// Minimum supported versions: +// FFmpeg: 1.1 +// libav: 9.20 (for Ubuntu 14.04) +#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(54, 35, 1) + #error Minimum supported version of libavcodec is 54.35.1. #endif -#if LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(54, 59, 106) - #error Minimum supported version of libavformat is 54.59.106. +#if LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(54, 20, 4) + #error Minimum supported version of libavformat is 54.20.4. #endif -#if LIBAVUTIL_VERSION_INT < AV_VERSION_INT(52, 13, 100) - #error Minimum supported version of libavutil is 52.13.100. +#if LIBAVUTIL_VERSION_INT < AV_VERSION_INT(52, 3, 0) + #error Minimum supported version of libavutil is 52.3.0. #endif ConfigureDef(config_ffmpeg); From e9ae7dcc407d8198a701b761d0431f456c8ad925 Mon Sep 17 00:00:00 2001 From: rdb Date: Mon, 2 Apr 2018 23:16:03 +0200 Subject: [PATCH 02/17] Use vector_uchar instead of string for binary network/file data This distinction allows us to better support Python 3, since it will raise exceptions when trying to put arbitrary binary data in a str object. This also adds some convenience functions for efficiently initializing a Datagram or PTA_uchar from a vector_uchar. --- .../src/distributed/cConnectionRepository.cxx | 6 +- dtool/src/prc/encryptStreamBuf.cxx | 7 ++- dtool/src/prc/streamReader.cxx | 12 ++-- dtool/src/prc/streamReader.h | 3 +- panda/src/downloader/downloadDb.cxx | 55 ++++++---------- panda/src/downloader/downloadDb.h | 8 +-- panda/src/downloader/socketStream.cxx | 19 +++--- panda/src/downloader/socketStream.h | 3 +- panda/src/express/datagram.I | 62 +++---------------- panda/src/express/datagram.h | 13 +--- panda/src/express/datagramIterator.I | 11 ++-- panda/src/express/datagramIterator.cxx | 14 ++--- panda/src/express/datagramIterator.h | 4 +- panda/src/express/hashVal.cxx | 6 +- panda/src/express/hashVal.h | 4 +- panda/src/express/multifile.cxx | 6 +- panda/src/express/pointerToArray.I | 26 ++++++++ panda/src/express/pointerToArray.h | 2 + panda/src/express/pointerToArrayBase.I | 10 +++ panda/src/express/pointerToArrayBase.h | 1 + panda/src/gobj/texture.cxx | 4 +- panda/src/movies/wavAudioCursor.cxx | 11 ++-- panda/src/recorder/socketStreamRecorder.cxx | 6 +- pandatool/src/flt/fltBeadID.cxx | 12 ++-- pandatool/src/flt/fltRecord.cxx | 5 +- 25 files changed, 143 insertions(+), 167 deletions(-) diff --git a/direct/src/distributed/cConnectionRepository.cxx b/direct/src/distributed/cConnectionRepository.cxx index 67d6ec1658..e247c2bd55 100644 --- a/direct/src/distributed/cConnectionRepository.cxx +++ b/direct/src/distributed/cConnectionRepository.cxx @@ -803,8 +803,9 @@ handle_update_field_owner() { Py_DECREF(dclass_this); // check if we should forward this update to the owner view + vector_uchar data = _di.get_remaining_bytes(); DCPacker packer; - packer.set_unpack_data(_di.get_remaining_bytes()); + packer.set_unpack_data((const char *)data.data(), data.size(), false); int field_id = packer.raw_unpack_uint16(); DCField *field = dclass->get_field_by_index(field_id); if (field->is_ownrecv()) { @@ -845,8 +846,9 @@ handle_update_field_owner() { Py_DECREF(dclass_this); // check if we should forward this update to the owner view + vector_uchar data = _di.get_remaining_bytes(); DCPacker packer; - packer.set_unpack_data(_di.get_remaining_bytes()); + packer.set_unpack_data((const char *)data.data(), data.size(), false); int field_id = packer.raw_unpack_uint16(); DCField *field = dclass->get_field_by_index(field_id); if (true) {//field->is_broadcast()) { diff --git a/dtool/src/prc/encryptStreamBuf.cxx b/dtool/src/prc/encryptStreamBuf.cxx index 21a33f93c0..dc38ac5894 100644 --- a/dtool/src/prc/encryptStreamBuf.cxx +++ b/dtool/src/prc/encryptStreamBuf.cxx @@ -146,14 +146,15 @@ open_read(istream *source, bool owns_source, const string &password) { int iv_length = EVP_CIPHER_iv_length(cipher); _read_block_size = EVP_CIPHER_block_size(cipher); - string iv = sr.extract_bytes(iv_length); + unsigned char *iv = (unsigned char *)alloca(iv_length); + iv_length = (int)sr.extract_bytes(iv, iv_length); _read_ctx = EVP_CIPHER_CTX_new(); nassertv(_read_ctx != NULL); // Initialize the context int result; - result = EVP_DecryptInit(_read_ctx, cipher, NULL, (unsigned char *)iv.data()); + result = EVP_DecryptInit(_read_ctx, cipher, NULL, (unsigned char *)iv); nassertv(result > 0); result = EVP_CIPHER_CTX_set_key_length(_read_ctx, key_length); @@ -170,7 +171,7 @@ open_read(istream *source, bool owns_source, const string &password) { unsigned char *key = (unsigned char *)alloca(key_length); result = PKCS5_PBKDF2_HMAC_SHA1((const char *)password.data(), password.length(), - (unsigned char *)iv.data(), iv.length(), + iv, iv_length, count * iteration_count_factor + 1, key_length, key); nassertv(result > 0); diff --git a/dtool/src/prc/streamReader.cxx b/dtool/src/prc/streamReader.cxx index 17252f5dd2..e9fa173ad1 100644 --- a/dtool/src/prc/streamReader.cxx +++ b/dtool/src/prc/streamReader.cxx @@ -117,16 +117,18 @@ extract_bytes(unsigned char *into, size_t size) { * Extracts the indicated number of bytes in the stream and returns them as a * string. Returns empty string at end-of-file. */ -string StreamReader:: +vector_uchar StreamReader:: extract_bytes(size_t size) { + vector_uchar buffer; if (_in->eof() || _in->fail()) { - return string(); + return buffer; } - char *buffer = (char *)alloca(size); - _in->read(buffer, size); + buffer.resize(size); + _in->read((char *)&buffer[0], size); size_t read_bytes = _in->gcount(); - return string(buffer, read_bytes); + buffer.resize(read_bytes); + return buffer; } /** diff --git a/dtool/src/prc/streamReader.h b/dtool/src/prc/streamReader.h index bedaa0eeb5..1643dd4a9c 100644 --- a/dtool/src/prc/streamReader.h +++ b/dtool/src/prc/streamReader.h @@ -19,6 +19,7 @@ #include "numeric_types.h" #include "littleEndian.h" #include "bigEndian.h" +#include "vector_uchar.h" /** * A class to read sequential binary data directly from an istream. Its @@ -71,7 +72,7 @@ PUBLISHED: EXTENSION(BLOCKING PyObject *readlines()); public: - BLOCKING string extract_bytes(size_t size); + BLOCKING vector_uchar extract_bytes(size_t size); BLOCKING string readline(); private: diff --git a/panda/src/downloader/downloadDb.cxx b/panda/src/downloader/downloadDb.cxx index f2a734af17..a210fba8bd 100644 --- a/panda/src/downloader/downloadDb.cxx +++ b/panda/src/downloader/downloadDb.cxx @@ -583,9 +583,7 @@ add_multifile_record(PT(MultifileRecord) mfr) { * Verifies magic number, returns the number of multifiles or -1 if invalid */ int DownloadDb::Db:: -parse_header(const string &data) { - Datagram dg(data); - +parse_header(Datagram dg) { // Make sure we have a good header DatagramIterator di(dg); uint32_t magic_number = di.get_uint32(); @@ -623,8 +621,7 @@ parse_header(const string &data) { * record */ int DownloadDb::Db:: -parse_record_header(const string &data) { - Datagram dg(data); +parse_record_header(Datagram dg) { DatagramIterator di(dg); int32_t record_length = di.get_int32(); downloader_cat.spam() @@ -639,14 +636,12 @@ parse_record_header(const string &data) { * Parses a multifile record (mfr) and returns one */ PT(DownloadDb::MultifileRecord) DownloadDb::Db:: -parse_mfr(const string &data) { +parse_mfr(Datagram dg) { PT(DownloadDb::MultifileRecord) mfr = new DownloadDb::MultifileRecord; - Datagram dg(data); DatagramIterator di(dg); - int32_t mfr_name_length = di.get_int32(); - mfr->_name = di.extract_bytes(mfr_name_length); + mfr->_name = di.get_string32(); mfr->_phase = di.get_float64(); mfr->_size = di.get_int32(); mfr->_status = di.get_int32(); @@ -676,14 +671,12 @@ parse_mfr(const string &data) { * Parses a file record (fr) and returns one */ PT(DownloadDb::FileRecord) DownloadDb::Db:: -parse_fr(const string &data) { +parse_fr(Datagram dg) { PT(DownloadDb::FileRecord) fr = new DownloadDb::FileRecord; - Datagram dg(data); DatagramIterator di(dg); - int32_t fr_name_length = di.get_int32(); - fr->_name = di.extract_bytes(fr_name_length); + fr->_name = di.get_string32(); // At one time, we stored files in the database with a backslash separator. // Nowadays we use a forward slash, but we should make sure we properly @@ -706,15 +699,14 @@ parse_fr(const string &data) { bool DownloadDb::Db:: read(StreamReader &sr, bool want_server_info) { // Read the header - string header; - header = sr.extract_bytes(_header_length); + vector_uchar header = sr.extract_bytes(_header_length); if (header.size() != (size_t)_header_length) { downloader_cat.error() << "truncated db file" << endl; return false; } // Parse the header - int num_multifiles = parse_header(header); + int num_multifiles = parse_header(Datagram(move(header))); if (num_multifiles < 0) { downloader_cat.error() << "invalid db header" << endl; return false; @@ -727,26 +719,26 @@ read(StreamReader &sr, bool want_server_info) { // of the record int mfr_header_length = sizeof(int32_t); - string mfr_header = sr.extract_bytes(mfr_header_length); + vector_uchar mfr_header = sr.extract_bytes(mfr_header_length); if (mfr_header.size() != (size_t)mfr_header_length) { downloader_cat.error() << "invalid mfr header" << endl; return false; } // Parse the header - int mfr_length = parse_record_header(mfr_header); + int mfr_length = parse_record_header(Datagram(move(mfr_header))); // Ok, now that we know the size of the mfr, read it in Make a buffer to // read the multifile record into do not count the header length twice int read_length = (mfr_length - mfr_header_length); - string mfr_record = sr.extract_bytes(read_length); + vector_uchar mfr_record = sr.extract_bytes(read_length); if (mfr_record.size() != (size_t)read_length) { downloader_cat.error() << "invalid mfr record" << endl; return false; } // Parse the mfr - PT(DownloadDb::MultifileRecord) mfr = parse_mfr(mfr_record); + PT(DownloadDb::MultifileRecord) mfr = parse_mfr(Datagram(move(mfr_record))); // Only read in the individual file info if you are the server if (want_server_info) { @@ -758,27 +750,27 @@ read(StreamReader &sr, bool want_server_info) { int fr_header_length = sizeof(int32_t); // Read the header - string fr_header = sr.extract_bytes(fr_header_length); + vector_uchar fr_header = sr.extract_bytes(fr_header_length); if (fr_header.size() != (size_t)fr_header_length) { downloader_cat.error() << "invalid fr header" << endl; return false; } // Parse the header - int fr_length = parse_record_header(fr_header); + int fr_length = parse_record_header(Datagram(move(fr_header))); // Ok, now that we know the size of the mfr, read it in do not count // the header length twice int read_length = (fr_length - fr_header_length); - string fr_record = sr.extract_bytes(read_length); + vector_uchar fr_record = sr.extract_bytes(read_length); if (fr_record.size() != (size_t)read_length) { downloader_cat.error() << "invalid fr record" << endl; return false; } // Parse the file record - PT(DownloadDb::FileRecord) fr = parse_fr(fr_record); + PT(DownloadDb::FileRecord) fr = parse_fr(Datagram(move(fr_record))); // Add this file record to the current multifilerecord mfr->add_file_record(fr); @@ -900,12 +892,10 @@ write_header(ostream &write_stream) { // Write the number of multifiles dg.add_int32(get_num_multifiles()); - string msg = dg.get_message(); - // Seek back to the beginning of the write stream write_stream.seekp(0); // Overwrite the old bogus header with the real header - write_stream.write(msg.data(), msg.length()); + write_stream.write((const char *)dg.get_data(), dg.get_length()); return true; } @@ -1100,17 +1090,8 @@ read_version_map(StreamReader &sr) { for (int i = 0; i < num_entries; i++) { - // Get the length of the file name - int name_length = sr.get_int32(); - if (sr.get_istream()->fail()) { - return false; - } - downloader_cat.spam() - << "DownloadDb::read_version_map() - name length: " << name_length - << endl; - // Get the file name - string name = sr.extract_bytes(name_length); + string name = sr.get_string32(); downloader_cat.spam() << "DownloadDb::read_version_map() - name: " << name << endl; diff --git a/panda/src/downloader/downloadDb.h b/panda/src/downloader/downloadDb.h index 02adedb3d6..b7f206c3b6 100644 --- a/panda/src/downloader/downloadDb.h +++ b/panda/src/downloader/downloadDb.h @@ -171,10 +171,10 @@ public: bool multifile_exists(string mfname) const; PT(MultifileRecord) get_multifile_record_named(string mfname) const; void add_multifile_record(PT(MultifileRecord) mfr); - int parse_header(const string &data); - int parse_record_header(const string &data); - PT(MultifileRecord) parse_mfr(const string &data); - PT(FileRecord) parse_fr(const string &data); + int parse_header(Datagram dg); + int parse_record_header(Datagram dg); + PT(MultifileRecord) parse_mfr(Datagram dg); + PT(FileRecord) parse_fr(Datagram dg); bool read(StreamReader &sr, bool want_server_info); bool write(StreamWriter &sw, bool want_server_info); Filename _filename; diff --git a/panda/src/downloader/socketStream.cxx b/panda/src/downloader/socketStream.cxx index af43847fb5..4e441f8f2b 100644 --- a/panda/src/downloader/socketStream.cxx +++ b/panda/src/downloader/socketStream.cxx @@ -50,17 +50,17 @@ SSReader:: bool SSReader:: do_receive_datagram(Datagram &dg) { if (_tcp_header_size == 0) { - _data_expected = _data_so_far.length(); + _data_expected = _data_so_far.size(); } if (_data_expected == 0) { // Read the first two bytes: the datagram length. - while ((int)_data_so_far.length() < _tcp_header_size) { + while ((int)_data_so_far.size() < _tcp_header_size) { int ch = _istream->get(); if (_istream->eof() || _istream->fail()) { _istream->clear(); return false; } - _data_so_far += (char)ch; + _data_so_far.push_back((unsigned char)ch); } Datagram header(_data_so_far); @@ -70,7 +70,7 @@ do_receive_datagram(Datagram &dg) { } else if (_tcp_header_size == 4) { _data_expected = di.get_uint32(); } - _data_so_far = _data_so_far.substr(_tcp_header_size); + _data_so_far.erase(_data_so_far.begin(), _data_so_far.begin() + _tcp_header_size); if (_data_expected == 0) { // Empty datagram. @@ -84,20 +84,19 @@ do_receive_datagram(Datagram &dg) { static const size_t buffer_size = 1024; char buffer[buffer_size]; - size_t read_count = min(_data_expected - _data_so_far.length(), - buffer_size); + size_t read_count = min(_data_expected - _data_so_far.size(), buffer_size); _istream->read(buffer, read_count); size_t count = _istream->gcount(); while (count != 0) { - _data_so_far.append(buffer, count); + _data_so_far.insert(_data_so_far.end(), buffer, buffer + count); - read_count = min(_data_expected - _data_so_far.length(), + read_count = min(_data_expected - _data_so_far.size(), buffer_size); _istream->read(buffer, read_count); count = _istream->gcount(); } - if (_data_so_far.length() < _data_expected) { + if (_data_so_far.size() < _data_expected) { // Not yet here. Clear the istream error flag and return false to // indicate more coming. _istream->clear(); @@ -108,7 +107,7 @@ do_receive_datagram(Datagram &dg) { dg.append_data(_data_so_far); _data_expected = 0; - _data_so_far = string(); + _data_so_far.clear(); return true; } diff --git a/panda/src/downloader/socketStream.h b/panda/src/downloader/socketStream.h index 36065e42af..84ce907a5e 100644 --- a/panda/src/downloader/socketStream.h +++ b/panda/src/downloader/socketStream.h @@ -21,6 +21,7 @@ #include "pdeque.h" #include "typedReferenceCount.h" #include "pointerTo.h" +#include "vector_uchar.h" // At the present, this module is not compiled if OpenSSL is not available, // since the only current use for it is to implement OpenSSL-defined @@ -54,7 +55,7 @@ private: istream *_istream; size_t _data_expected; - string _data_so_far; + vector_uchar _data_so_far; int _tcp_header_size; #ifdef SIMULATE_NETWORK_DELAY diff --git a/panda/src/express/datagram.I b/panda/src/express/datagram.I index fa6063805b..3bd7812969 100644 --- a/panda/src/express/datagram.I +++ b/panda/src/express/datagram.I @@ -42,58 +42,16 @@ Datagram(const void *data, size_t size) : * Constructs a datagram from an existing block of data. */ INLINE Datagram:: -Datagram(const string &data) : +Datagram(vector_uchar data) : + _data(move(data)), #ifdef STDFLOAT_DOUBLE _stdfloat_double(true) #else _stdfloat_double(false) #endif { - append_data(data); } -/** - * - */ -INLINE Datagram:: -Datagram(const Datagram ©) : - _data(copy._data), - _stdfloat_double(copy._stdfloat_double) -{ -} - -/** - * - */ -INLINE void Datagram:: -operator = (const Datagram ©) { - _data = copy._data; - _stdfloat_double = copy._stdfloat_double; -} - -#ifdef USE_MOVE_SEMANTICS -/** - * - */ -INLINE Datagram:: -Datagram(Datagram &&from) NOEXCEPT : - _data(move(from._data)), - _stdfloat_double(from._stdfloat_double) -{ -} -#endif // USE_MOVE_SEMANTICS - -#ifdef USE_MOVE_SEMANTICS -/** - * - */ -INLINE void Datagram:: -operator = (Datagram &&from) NOEXCEPT { - _data = move(from._data); - _stdfloat_double = from._stdfloat_double; -} -#endif // USE_MOVE_SEMANTICS - /** * Adds a boolean value to the datagram. */ @@ -291,7 +249,7 @@ add_string(const string &str) { add_uint16((uint16_t)str.length()); // Add the string - append_data(str); + append_data(str.data(), str.length()); } /** @@ -304,18 +262,18 @@ add_string32(const string &str) { add_uint32((uint32_t)str.length()); // Add the string - append_data(str); + append_data(str.data(), str.length()); } /** * Adds a variable-length string to the datagram, as a NULL-terminated string. */ INLINE void Datagram:: -add_z_string(string str) { +add_z_string(const string &str) { // We must not have any nested null characters in the string. size_t null_pos = str.find('\0'); // Add the string (sans the null character). - append_data(str.substr(0, null_pos)); + append_data(str.data(), std::min(null_pos, str.length())); // And the null character. add_uint8('\0'); @@ -329,11 +287,11 @@ add_z_string(string str) { INLINE void Datagram:: add_fixed_string(const string &str, size_t size) { if (str.length() < size) { - append_data(str); + append_data(str.data(), str.length()); pad_bytes(size - str.length()); } else { // str.length() >= size - append_data(str.substr(0, size)); + append_data(str.data(), size); } } @@ -341,8 +299,8 @@ add_fixed_string(const string &str, size_t size) { * Appends some more raw data to the end of the datagram. */ INLINE void Datagram:: -append_data(const string &data) { - append_data(data.data(), data.length()); +append_data(const vector_uchar &data) { + append_data(data.data(), data.size()); } /** diff --git a/panda/src/express/datagram.h b/panda/src/express/datagram.h index c2cb3b4c8b..9250ac89d3 100644 --- a/panda/src/express/datagram.h +++ b/panda/src/express/datagram.h @@ -39,14 +39,7 @@ class EXPCL_PANDAEXPRESS Datagram : public TypedObject { PUBLISHED: INLINE Datagram(); INLINE Datagram(const void *data, size_t size); - INLINE Datagram(const string &data); - INLINE Datagram(const Datagram ©); - INLINE void operator = (const Datagram ©); - -#ifdef USE_MOVE_SEMANTICS - INLINE Datagram(Datagram &&from) NOEXCEPT; - INLINE void operator = (Datagram &&from) NOEXCEPT; -#endif + INLINE explicit Datagram(vector_uchar data); virtual ~Datagram(); @@ -80,13 +73,13 @@ PUBLISHED: INLINE void add_string(const string &str); INLINE void add_string32(const string &str); - INLINE void add_z_string(string str); + INLINE void add_z_string(const string &str); INLINE void add_fixed_string(const string &str, size_t size); void add_wstring(const wstring &str); void pad_bytes(size_t size); void append_data(const void *data, size_t size); - INLINE void append_data(const string &data); + INLINE void append_data(const vector_uchar &data); void assign(const void *data, size_t size); diff --git a/panda/src/express/datagramIterator.I b/panda/src/express/datagramIterator.I index cefde399f2..42c25eeed4 100644 --- a/panda/src/express/datagramIterator.I +++ b/panda/src/express/datagramIterator.I @@ -422,14 +422,13 @@ skip_bytes(size_t size) { * Returns the remaining bytes in the datagram as a string, but does not * extract them from the iterator. */ -INLINE string DatagramIterator:: +INLINE vector_uchar DatagramIterator:: get_remaining_bytes() const { - nassertr(_datagram != (const Datagram *)NULL, ""); - nassertr(_current_index <= _datagram->get_length(), ""); + nassertr(_datagram != (const Datagram *)NULL, vector_uchar()); + nassertr(_current_index <= _datagram->get_length(), vector_uchar()); - const char *ptr = (const char *)_datagram->get_data(); - size_t remaining_size = _datagram->get_length() - _current_index; - return string(ptr + _current_index, remaining_size); + const unsigned char *ptr = (const unsigned char *)_datagram->get_data(); + return vector_uchar(ptr + _current_index, ptr + _datagram->get_length()); } /** diff --git a/panda/src/express/datagramIterator.cxx b/panda/src/express/datagramIterator.cxx index 22ac9ecb19..880719f32b 100644 --- a/panda/src/express/datagramIterator.cxx +++ b/panda/src/express/datagramIterator.cxx @@ -119,18 +119,18 @@ get_wstring() { * Extracts the indicated number of bytes in the datagram and returns them as * a string. */ -string DatagramIterator:: +vector_uchar DatagramIterator:: extract_bytes(size_t size) { - nassertr((int)size >= 0, ""); - nassertr(_datagram != (const Datagram *)NULL, ""); - nassertr(_current_index + size <= _datagram->get_length(), ""); + nassertr((int)size >= 0, vector_uchar()); + nassertr(_datagram != (const Datagram *)NULL, vector_uchar()); + nassertr(_current_index + size <= _datagram->get_length(), vector_uchar()); - const char *ptr = (const char *)_datagram->get_data(); - size_t last_index = _current_index; + const unsigned char *ptr = (const unsigned char *)_datagram->get_data(); + ptr += _current_index; _current_index += size; - return string(ptr + last_index, size); + return vector_uchar(ptr, ptr + size); } /** diff --git a/panda/src/express/datagramIterator.h b/panda/src/express/datagramIterator.h index 7126901324..ec76be3f07 100644 --- a/panda/src/express/datagramIterator.h +++ b/panda/src/express/datagramIterator.h @@ -62,10 +62,10 @@ PUBLISHED: wstring get_wstring(); INLINE void skip_bytes(size_t size); - string extract_bytes(size_t size); + vector_uchar extract_bytes(size_t size); size_t extract_bytes(unsigned char *into, size_t size); - INLINE string get_remaining_bytes() const; + INLINE vector_uchar get_remaining_bytes() const; INLINE size_t get_remaining_size() const; INLINE const Datagram &get_datagram() const; diff --git a/panda/src/express/hashVal.cxx b/panda/src/express/hashVal.cxx index 7d52dc5a5e..aa08714668 100644 --- a/panda/src/express/hashVal.cxx +++ b/panda/src/express/hashVal.cxx @@ -143,11 +143,11 @@ set_from_hex(const string &text) { /** * Returns the HashVal as a 16-byte binary string. */ -string HashVal:: +vector_uchar HashVal:: as_bin() const { Datagram dg; write_datagram(dg); - return dg.get_message(); + return vector_uchar((unsigned char *)dg.get_data(), (unsigned char *)dg.get_data() + dg.get_length()); } /** @@ -155,7 +155,7 @@ as_bin() const { * false otherwise. */ bool HashVal:: -set_from_bin(const string &text) { +set_from_bin(const vector_uchar &text) { nassertr(text.size() == 16, false); Datagram dg(text); DatagramIterator dgi(dg); diff --git a/panda/src/express/hashVal.h b/panda/src/express/hashVal.h index d0652d1e01..a4b3754614 100644 --- a/panda/src/express/hashVal.h +++ b/panda/src/express/hashVal.h @@ -55,8 +55,8 @@ PUBLISHED: string as_hex() const; bool set_from_hex(const string &text); - string as_bin() const; - bool set_from_bin(const string &text); + vector_uchar as_bin() const; + bool set_from_bin(const vector_uchar &text); INLINE void write_datagram(Datagram &destination) const; INLINE void read_datagram(DatagramIterator &source); diff --git a/panda/src/express/multifile.cxx b/panda/src/express/multifile.cxx index 0d98849efe..5a7d86f17b 100644 --- a/panda/src/express/multifile.cxx +++ b/panda/src/express/multifile.cxx @@ -2402,7 +2402,7 @@ check_signatures() { nassertv(stream != NULL); StreamReader reader(*stream); size_t sig_size = reader.get_uint32(); - string sig_string = reader.extract_bytes(sig_size); + vector_uchar sig_data = reader.extract_bytes(sig_size); size_t num_certs = reader.get_uint32(); @@ -2470,9 +2470,7 @@ check_signatures() { // Now check that the signature matches the hash. int verify_result = - EVP_VerifyFinal(md_ctx, - (unsigned char *)sig_string.data(), - sig_string.size(), pkey); + EVP_VerifyFinal(md_ctx, sig_data.data(), sig_data.size(), pkey); if (verify_result == 1) { // The signature matches; save the certificate and its chain. _signatures.push_back(chain); diff --git a/panda/src/express/pointerToArray.I b/panda/src/express/pointerToArray.I index c3a10f3737..535de63d01 100644 --- a/panda/src/express/pointerToArray.I +++ b/panda/src/express/pointerToArray.I @@ -92,6 +92,19 @@ PointerToArray(PointerToArray &&from) NOEXCEPT : } #endif // USE_MOVE_SEMANTICS +#ifdef USE_MOVE_SEMANTICS +/** + * Initializes the PTA from a vector. + */ +template +INLINE PointerToArray:: +PointerToArray(pvector &&from, TypeHandle type_handle) : + PointerToArrayBase(new ReferenceCountedVector(move(from))), + _type_handle(type_handle) +{ +} +#endif // USE_MOVE_SEMANTICS + /** * */ @@ -710,6 +723,19 @@ ConstPointerToArray(ConstPointerToArray &&from) NOEXCEPT : } #endif // USE_MOVE_SEMANTICS +#ifdef USE_MOVE_SEMANTICS +/** + * Initializes the PTA from a vector. + */ +template +INLINE ConstPointerToArray:: +ConstPointerToArray(pvector &&from, TypeHandle type_handle) : + PointerToArrayBase(new ReferenceCountedVector(move(from))), + _type_handle(type_handle) +{ +} +#endif // USE_MOVE_SEMANTICS + /** * */ diff --git a/panda/src/express/pointerToArray.h b/panda/src/express/pointerToArray.h index 762a6cb8ce..4e8a052e0f 100644 --- a/panda/src/express/pointerToArray.h +++ b/panda/src/express/pointerToArray.h @@ -141,6 +141,7 @@ public: #ifdef USE_MOVE_SEMANTICS INLINE PointerToArray(PointerToArray &&from) NOEXCEPT; + INLINE explicit PointerToArray(pvector &&from, TypeHandle type_handle = get_type_handle(Element)); #endif public: @@ -302,6 +303,7 @@ PUBLISHED: #ifdef USE_MOVE_SEMANTICS INLINE ConstPointerToArray(PointerToArray &&from) NOEXCEPT; INLINE ConstPointerToArray(ConstPointerToArray &&from) NOEXCEPT; + INLINE explicit ConstPointerToArray(pvector &&from, TypeHandle type_handle = get_type_handle(Element)); #endif // Duplicating the interface of vector. diff --git a/panda/src/express/pointerToArrayBase.I b/panda/src/express/pointerToArrayBase.I index 89acba1206..de8c17d843 100644 --- a/panda/src/express/pointerToArrayBase.I +++ b/panda/src/express/pointerToArrayBase.I @@ -39,6 +39,16 @@ ReferenceCountedVector(const Element *begin, const Element *end, TypeHandle type { } +/** + * Creates an array that takes its elements from the given vector. + */ +template +INLINE ReferenceCountedVector:: +ReferenceCountedVector(pvector &&from) : + pvector(move(from)) +{ +} + /** * */ diff --git a/panda/src/express/pointerToArrayBase.h b/panda/src/express/pointerToArrayBase.h index 3220ca4c70..8c33f69be2 100644 --- a/panda/src/express/pointerToArrayBase.h +++ b/panda/src/express/pointerToArrayBase.h @@ -43,6 +43,7 @@ public: INLINE ReferenceCountedVector(TypeHandle type_handle); INLINE ReferenceCountedVector(size_type initial_size, TypeHandle type_handle); INLINE ReferenceCountedVector(const Element *begin, const Element *end, TypeHandle type_handle); + INLINE ReferenceCountedVector(pvector &&from); ALLOC_DELETED_CHAIN(ReferenceCountedVector); INLINE size_type size() const; diff --git a/panda/src/gobj/texture.cxx b/panda/src/gobj/texture.cxx index c4a8149ab4..5328b921d9 100644 --- a/panda/src/gobj/texture.cxx +++ b/panda/src/gobj/texture.cxx @@ -4207,7 +4207,9 @@ bool Texture:: do_read_ktx(CData *cdata, istream &in, const string &filename, bool header_only) { StreamReader ktx(in); - if (ktx.extract_bytes(12) != "\xABKTX 11\xBB\r\n\x1A\n") { + unsigned char magic[12]; + if (ktx.extract_bytes(magic, 12) != 12 || + memcmp(magic, "\xABKTX 11\xBB\r\n\x1A\n", 12) != 0) { gobj_cat.error() << filename << " is not a KTX file.\n"; return false; diff --git a/panda/src/movies/wavAudioCursor.cxx b/panda/src/movies/wavAudioCursor.cxx index 3b220749c4..ccfaa9733e 100644 --- a/panda/src/movies/wavAudioCursor.cxx +++ b/panda/src/movies/wavAudioCursor.cxx @@ -106,7 +106,8 @@ WavAudioCursor(WavAudio *src, istream *stream) : nassertv(stream != NULL); // Beginning of "RIFF" chunk. - if (_reader.extract_bytes(4) != "RIFF") { + unsigned char magic[4]; + if (_reader.extract_bytes(magic, 4) != 4 || memcmp(magic, "RIFF", 4) != 0) { movies_cat.error() << ".wav file is not a valid RIFF file.\n"; return; @@ -114,7 +115,7 @@ WavAudioCursor(WavAudio *src, istream *stream) : unsigned int chunk_size = _reader.get_uint32(); - if (_reader.extract_bytes(4) != "WAVE") { + if (_reader.extract_bytes(magic, 4) != 4 || memcmp(magic, "WAVE", 4) != 0) { movies_cat.error() << ".wav file is a RIFF file but does not start with a WAVE chunk.\n"; return; @@ -126,10 +127,10 @@ WavAudioCursor(WavAudio *src, istream *stream) : while ((!have_fmt || !have_data) && _stream->good() && (bytes_read + 8) < chunk_size) { - string subchunk_id = _reader.extract_bytes(4); + _reader.extract_bytes(magic, 4); unsigned int subchunk_size = _reader.get_uint32(); - if (subchunk_id == "fmt ") { + if (memcmp(magic, "fmt ", 4) == 0) { // The format chunk specifies information about the storage. nassertv(subchunk_size >= 16); have_fmt = true; @@ -202,7 +203,7 @@ WavAudioCursor(WavAudio *src, istream *stream) : _reader.skip_bytes(subchunk_size - read_bytes); } - } else if (subchunk_id == "data") { + } else if (memcmp(magic, "data", 4) == 0) { // The data chunk contains the actual sammples. if (!have_fmt) { movies_cat.error() diff --git a/panda/src/recorder/socketStreamRecorder.cxx b/panda/src/recorder/socketStreamRecorder.cxx index 1b9493fc03..6209239044 100644 --- a/panda/src/recorder/socketStreamRecorder.cxx +++ b/panda/src/recorder/socketStreamRecorder.cxx @@ -82,8 +82,10 @@ play_frame(DatagramIterator &scan, BamReader *manager) { int num_packets = scan.get_uint16(); for (int i = 0; i < num_packets; i++) { - string packet = scan.get_string(); - _data.push_back(Datagram(packet)); + size_t size = scan.get_uint16(); + vector_uchar packet(size); + scan.extract_bytes(&packet[0], size); + _data.push_back(Datagram(move(packet))); } } diff --git a/pandatool/src/flt/fltBeadID.cxx b/pandatool/src/flt/fltBeadID.cxx index 6346d6282e..f55d3e88cc 100644 --- a/pandatool/src/flt/fltBeadID.cxx +++ b/pandatool/src/flt/fltBeadID.cxx @@ -78,9 +78,8 @@ extract_record(FltRecordReader &reader) { bool FltBeadID:: extract_ancillary(FltRecordReader &reader) { if (reader.get_opcode() == FO_long_id) { - string s = reader.get_iterator().get_remaining_bytes(); - size_t zero_byte = s.find('\0'); - _id = s.substr(0, zero_byte); + vector_uchar s = reader.get_iterator().get_remaining_bytes(); + _id.assign((const char *)s.data(), strnlen((const char *)s.data(), s.size())); return true; } @@ -109,14 +108,11 @@ build_record(FltRecordWriter &writer) const { FltError FltBeadID:: write_ancillary(FltRecordWriter &writer) const { if (_id.length() > 7) { + Datagram dc; // Although the manual mentions nothing of this, it is essential that the // length of the record be a multiple of 4 bytes. - string id = _id; - while ((id.length() % 4) != 0) { - id += '\0'; - } - Datagram dc(id); + dc.add_fixed_string(_id, (_id.length() + 3) & ~3); FltError result = writer.write_record(FO_long_id, dc); if (result != FE_ok) { diff --git a/pandatool/src/flt/fltRecord.cxx b/pandatool/src/flt/fltRecord.cxx index c3dc21d070..a0af83e180 100644 --- a/pandatool/src/flt/fltRecord.cxx +++ b/pandatool/src/flt/fltRecord.cxx @@ -621,7 +621,8 @@ extract_record(FltRecordReader &) { bool FltRecord:: extract_ancillary(FltRecordReader &reader) { if (reader.get_opcode() == FO_comment) { - _comment = reader.get_iterator().get_remaining_bytes(); + vector_uchar s = reader.get_iterator().get_remaining_bytes(); + _comment.assign((const char *)s.data(), strnlen((const char *)s.data(), s.size())); return true; } @@ -735,7 +736,7 @@ build_record(FltRecordWriter &) const { FltError FltRecord:: write_ancillary(FltRecordWriter &writer) const { if (!_comment.empty()) { - Datagram dc(_comment); + Datagram dc(_comment.data(), _comment.size()); FltError result = writer.write_record(FO_comment, dc); if (result != FE_ok) { return result; From 7cbdd3b6c49312b90489ba7ebfce5b2b0f22910b Mon Sep 17 00:00:00 2001 From: rdb Date: Wed, 4 Apr 2018 21:36:10 +0200 Subject: [PATCH 03/17] gobj: release GIL for (un)compress_ram_image --- panda/src/gobj/texture.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/panda/src/gobj/texture.h b/panda/src/gobj/texture.h index 5816b8c538..890710a216 100644 --- a/panda/src/gobj/texture.h +++ b/panda/src/gobj/texture.h @@ -464,10 +464,10 @@ PUBLISHED: MAKE_PROPERTY(keep_ram_image, get_keep_ram_image, set_keep_ram_image); MAKE_PROPERTY(cacheable, is_cacheable); - INLINE bool compress_ram_image(CompressionMode compression = CM_on, - QualityLevel quality_level = QL_default, - GraphicsStateGuardianBase *gsg = NULL); - INLINE bool uncompress_ram_image(); + BLOCKING INLINE bool compress_ram_image(CompressionMode compression = CM_on, + QualityLevel quality_level = QL_default, + GraphicsStateGuardianBase *gsg = NULL); + BLOCKING INLINE bool uncompress_ram_image(); INLINE int get_num_ram_mipmap_images() const; INLINE bool has_ram_mipmap_image(int n) const; From 94476fd1f0e402a237f1b4cf5d77347a7ec4f682 Mon Sep 17 00:00:00 2001 From: rdb Date: Wed, 4 Apr 2018 21:46:01 +0200 Subject: [PATCH 04/17] glgsg: fix invalid operation error with multisample FBO Panda is forgetting to reset the current FBO when resolving multisamples. --- panda/src/glstuff/glGraphicsBuffer_src.cxx | 1 + 1 file changed, 1 insertion(+) diff --git a/panda/src/glstuff/glGraphicsBuffer_src.cxx b/panda/src/glstuff/glGraphicsBuffer_src.cxx index 43f3dbd809..35daa0b225 100644 --- a/panda/src/glstuff/glGraphicsBuffer_src.cxx +++ b/panda/src/glstuff/glGraphicsBuffer_src.cxx @@ -1767,6 +1767,7 @@ resolve_multisamples() { } glgsg->_glBindFramebuffer(GL_DRAW_FRAMEBUFFER_EXT, fbo); glgsg->_glBindFramebuffer(GL_READ_FRAMEBUFFER_EXT, _fbo_multisample); + glgsg->_current_fbo = fbo; // If the depth buffer is shared, resolve it only on the last to render FBO. bool do_depth_blit = false; From 552a649ef3fcad1690f99e53b1d50e7494bce21c Mon Sep 17 00:00:00 2001 From: rdb Date: Wed, 4 Apr 2018 21:48:02 +0200 Subject: [PATCH 05/17] interrogate: add back NULL definition, fixes some keyword arguments I am not sure why the definition of NULL was removed; it might have been by mistake, but in any case it broke code like this: img = PNMImage(w, h, color_space=CS_srgb) since it would not understand the default value (NULL) for the argument preceding color_space. --- dtool/src/parser-inc/stdtypedefs.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dtool/src/parser-inc/stdtypedefs.h b/dtool/src/parser-inc/stdtypedefs.h index c1febd3955..f971d7e1f2 100644 --- a/dtool/src/parser-inc/stdtypedefs.h +++ b/dtool/src/parser-inc/stdtypedefs.h @@ -41,6 +41,11 @@ inline namespace std { struct timeval; +#ifdef __cplusplus +#define NULL 0L +#else +#define NULL ((void *)0) +#endif typedef decltype(nullptr) nullptr_t; // One day, we might extend interrogate to be able to parse this, From 7125a3e5877edc056b6e076285b54f3869dce9f6 Mon Sep 17 00:00:00 2001 From: rdb Date: Thu, 5 Apr 2018 20:49:13 +0200 Subject: [PATCH 06/17] interrogate: allow passing None for args that have NULL as default value --- .../interfaceMakerPythonNative.cxx | 64 ++++++++++++++++--- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/dtool/src/interrogate/interfaceMakerPythonNative.cxx b/dtool/src/interrogate/interfaceMakerPythonNative.cxx index 96420ef663..5ae2d3fca9 100644 --- a/dtool/src/interrogate/interfaceMakerPythonNative.cxx +++ b/dtool/src/interrogate/interfaceMakerPythonNative.cxx @@ -5414,6 +5414,18 @@ write_function_instance(ostream &out, FunctionRemap *remap, parameter_list += ", &" + param_name; } + // If the default value is NULL, we also accept a None value. + bool maybe_none = false; + if (default_value != nullptr && (return_flags & RF_coerced) == 0) { + CPPExpression::Result res = param->get_default_value()->evaluate(); + if (res._type == CPPExpression::RT_integer || + res._type == CPPExpression::RT_pointer) { + if (res.as_integer() == 0) { + maybe_none = true; + } + } + } + string class_name = obj_type->get_local_name(&parser); // need to a forward scope for this class.. @@ -5464,17 +5476,27 @@ write_function_instance(ostream &out, FunctionRemap *remap, type->output_instance(extra_convert, param_name + "_this", &parser); - if (is_optional) { + if (is_optional && maybe_none) { + extra_convert + << default_expr << ";\n" + << "if (" << param_name << " != NULL && " << param_name << " != Py_None) {\n" + << " " << param_name << "_this"; + } else if (is_optional) { extra_convert << default_expr << ";\n" << "if (" << param_name << " != NULL) {\n" << " " << param_name << "_this"; + } else if (maybe_none) { + extra_convert + << " = NULL;\n" + << "if (" << param_name << " != Py_None) {\n" + << " " << param_name << "_this"; } extra_convert << " = Dtool_Coerce_" + make_safe_name(class_name) + "(" + param_name + ", " + param_name + "_local);\n"; - if (is_optional) { + if (is_optional || maybe_none) { extra_convert << "}\n"; } @@ -5485,8 +5507,12 @@ write_function_instance(ostream &out, FunctionRemap *remap, if (report_errors) { // We were asked to report any errors. Let's do it. - if (is_optional) { + if (is_optional && maybe_none) { + extra_convert << "if (" << param_name << " != NULL && " << param_name << " != Py_None && !" << coerce_call << ") {\n"; + } else if (is_optional) { extra_convert << "if (" << param_name << " != NULL && !" << coerce_call << ") {\n"; + } else if (maybe_none) { + extra_convert << "if (" << param_name << " != Py_None && !" << coerce_call << ") {\n"; } else { extra_convert << "if (!" << coerce_call << ") {\n"; } @@ -5509,19 +5535,35 @@ write_function_instance(ostream &out, FunctionRemap *remap, } extra_convert << "}\n"; + } else if (is_optional && maybe_none) { + extra_param_check << " && (" << param_name << " == NULL || " << param_name << " == Py_None || " << coerce_call << ")"; + } else if (is_optional) { extra_param_check << " && (" << param_name << " == NULL || " << coerce_call << ")"; + } else if (maybe_none) { + extra_param_check << " && (" << param_name << " == Py_None || " << coerce_call << ")"; + } else { extra_param_check << " && " << coerce_call; } - } else { + } else { // The regular, non-coercion case. type->output_instance(extra_convert, param_name + "_this", &parser); - if (is_optional) { + if (is_optional && maybe_none) { extra_convert << default_expr << ";\n" - << "if (" << param_name << " != (PyObject *)NULL) {\n" + << "if (" << param_name << " != NULL && " << param_name << " != Py_None) {\n" + << " " << param_name << "_this"; + } else if (is_optional) { + extra_convert + << default_expr << ";\n" + << "if (" << param_name << " != NULL) {\n" + << " " << param_name << "_this"; + } else if (maybe_none) { + extra_convert + << " = NULL;\n" + << "if (" << param_name << " != Py_None) {\n" << " " << param_name << "_this"; } if (const_ok && !report_errors) { @@ -5529,7 +5571,7 @@ write_function_instance(ostream &out, FunctionRemap *remap, // simpler. But maybe we should just reorganize these functions // entirely? extra_convert << " = NULL;\n"; - int indent_level = is_optional ? 2 : 0; + int indent_level = (is_optional || maybe_none) ? 2 : 0; indent(extra_convert, indent_level) << "DtoolInstance_GetPointer(" << param_name << ", " << param_name << "_this" @@ -5545,9 +5587,15 @@ write_function_instance(ostream &out, FunctionRemap *remap, << "\", " << const_ok << ", " << report_errors << ");\n"; } - if (is_optional) { + if (is_optional && maybe_none) { + extra_convert << "}\n"; + extra_param_check << " && (" << param_name << " == NULL || " << param_name << " == Py_None || " << param_name << "_this != NULL)"; + } else if (is_optional) { extra_convert << "}\n"; extra_param_check << " && (" << param_name << " == NULL || " << param_name << "_this != NULL)"; + } else if (maybe_none) { + extra_convert << "}\n"; + extra_param_check << " && (" << param_name << " == Py_None || " << param_name << "_this != NULL)"; } else { extra_param_check << " && " << param_name << "_this != NULL"; } From de6d753f79886ab45d8c3cec6b231d15afe64d94 Mon Sep 17 00:00:00 2001 From: rdb Date: Fri, 6 Apr 2018 17:16:18 +0200 Subject: [PATCH 07/17] net: don't use get_message() when constructing datagram header This is causing an unnecessary copy operation. --- panda/src/net/datagramTCPHeader.cxx | 13 ++++++------- panda/src/net/datagramUDPHeader.cxx | 15 ++++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/panda/src/net/datagramTCPHeader.cxx b/panda/src/net/datagramTCPHeader.cxx index 533e3e671b..bd013f0c33 100644 --- a/panda/src/net/datagramTCPHeader.cxx +++ b/panda/src/net/datagramTCPHeader.cxx @@ -24,23 +24,23 @@ */ DatagramTCPHeader:: DatagramTCPHeader(const NetDatagram &datagram, int header_size) { - const string &str = datagram.get_message(); + size_t length = datagram.get_length(); switch (header_size) { case 0: break; case datagram_tcp16_header_size: { - uint16_t size = str.length(); - nassertv(size == str.length()); + uint16_t size = (uint16_t)length; + nassertv((size_t)size == length); _header.add_uint16(size); } break; case datagram_tcp32_header_size: { - uint32_t size = str.length(); - nassertv(size == str.length()); + uint32_t size = (uint32_t)length; + nassertv((size_t)size == length); _header.add_uint32(size); } break; @@ -93,8 +93,7 @@ verify_datagram(const NetDatagram &datagram, int header_size) const { return true; } - const string &str = datagram.get_message(); - int actual_size = str.length(); + int actual_size = (int)datagram.get_length(); int expected_size = get_datagram_size(header_size); if (actual_size == expected_size) { return true; diff --git a/panda/src/net/datagramUDPHeader.cxx b/panda/src/net/datagramUDPHeader.cxx index b491190187..2191c0241a 100644 --- a/panda/src/net/datagramUDPHeader.cxx +++ b/panda/src/net/datagramUDPHeader.cxx @@ -24,10 +24,11 @@ */ DatagramUDPHeader:: DatagramUDPHeader(const NetDatagram &datagram) { - const string &str = datagram.get_message(); + const unsigned char *begin = (const unsigned char *)datagram.get_data(); + const unsigned char *end = begin + datagram.get_length(); uint16_t checksum = 0; - for (size_t p = 0; p < str.size(); p++) { - checksum += (uint16_t)(uint8_t)str[p]; + for (const unsigned char *p = begin; p != end; ++p) { + checksum += (uint16_t)(uint8_t)*p; } // Now pack the header. @@ -49,11 +50,11 @@ DatagramUDPHeader(const void *data) : _header(data, datagram_udp_header_size) { */ bool DatagramUDPHeader:: verify_datagram(const NetDatagram &datagram) const { - const string &str = datagram.get_message(); - + const unsigned char *begin = (const unsigned char *)datagram.get_data(); + const unsigned char *end = begin + datagram.get_length(); uint16_t checksum = 0; - for (size_t p = 0; p < str.size(); p++) { - checksum += (uint16_t)(uint8_t)str[p]; + for (const unsigned char *p = begin; p != end; ++p) { + checksum += (uint16_t)(uint8_t)*p; } if (checksum == get_datagram_checksum()) { From b4d29e609657e15a807da4832aaed20373b5ab26 Mon Sep 17 00:00:00 2001 From: rdb Date: Fri, 6 Apr 2018 17:41:12 +0200 Subject: [PATCH 08/17] express: allow using bytes() on Datagram Closes: #297 --- panda/src/express/datagram.I | 12 ++++++++++++ panda/src/express/datagram.h | 1 + tests/putil/test_datagram.py | 10 ++++++++++ 3 files changed, 23 insertions(+) diff --git a/panda/src/express/datagram.I b/panda/src/express/datagram.I index 3bd7812969..499bfab48b 100644 --- a/panda/src/express/datagram.I +++ b/panda/src/express/datagram.I @@ -316,6 +316,18 @@ get_message() const { } } +/** + * Returns the datagram's data as a bytes object. + */ +INLINE vector_uchar Datagram:: +__bytes__() const { + if (!_data.empty()) { + return vector_uchar(_data.v()); + } else { + return vector_uchar(); + } +} + /** * Returns a pointer to the beginning of the datagram's data. */ diff --git a/panda/src/express/datagram.h b/panda/src/express/datagram.h index 9250ac89d3..00a46ef36f 100644 --- a/panda/src/express/datagram.h +++ b/panda/src/express/datagram.h @@ -84,6 +84,7 @@ PUBLISHED: void assign(const void *data, size_t size); INLINE string get_message() const; + INLINE vector_uchar __bytes__() const; INLINE const void *get_data() const; INLINE size_t get_length() const; diff --git a/tests/putil/test_datagram.py b/tests/putil/test_datagram.py index 4a34bc88a2..ceadff67b8 100644 --- a/tests/putil/test_datagram.py +++ b/tests/putil/test_datagram.py @@ -76,6 +76,16 @@ def datagram_large(): return dg, readback_function +def test_datagram_bytes(): + """Tests that we can put and get a bytes object on Datagram.""" + dg = core.Datagram(b'abc\x00') + dg.append_data(b'\xff123') + assert bytes(dg) == b'abc\x00\xff123' + + dgi = core.DatagramIterator(dg) + dgi.get_remaining_bytes() == b'abc\x00\xff123' + + def test_iterator(datagram_small): """This tests Datagram/DatagramIterator, and sort of serves as a self-check of the test fixtures too.""" From 22f933a41970e285d6abc4fd6c5d19e50fd423de Mon Sep 17 00:00:00 2001 From: rdb Date: Fri, 6 Apr 2018 20:34:48 +0200 Subject: [PATCH 09/17] tests: disable datagram bytes() test on Python 2 See #297 --- tests/putil/test_datagram.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/putil/test_datagram.py b/tests/putil/test_datagram.py index ceadff67b8..1349bd5656 100644 --- a/tests/putil/test_datagram.py +++ b/tests/putil/test_datagram.py @@ -1,5 +1,6 @@ import pytest from panda3d import core +import sys # Fixtures for generating interesting datagrams (and verification functions) on # the fly... @@ -76,6 +77,7 @@ def datagram_large(): return dg, readback_function +@pytest.mark.skipif(sys.version_info < (3, 0), reason="Requires Python 3") def test_datagram_bytes(): """Tests that we can put and get a bytes object on Datagram.""" dg = core.Datagram(b'abc\x00') From d13464104c19be8cb5c41390a6db4aab58ca0817 Mon Sep 17 00:00:00 2001 From: rdb Date: Fri, 6 Apr 2018 20:49:43 +0200 Subject: [PATCH 10/17] Fix compilation issue with --nothing on MSVC I have no idea why this error happens, but it does not seem worth the effort to investigate further, so I'm just reverting the previous change to this file. --- dtool/src/dtoolbase/pvector.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dtool/src/dtoolbase/pvector.h b/dtool/src/dtoolbase/pvector.h index b146947543..14bd542f37 100644 --- a/dtool/src/dtoolbase/pvector.h +++ b/dtool/src/dtoolbase/pvector.h @@ -46,9 +46,24 @@ public: typedef TYPENAME base_class::size_type size_type; explicit pvector(TypeHandle type_handle = pvector_type_handle) : base_class(allocator(type_handle)) { } + pvector(const pvector ©) : base_class(copy) { } explicit pvector(size_type n, TypeHandle type_handle = pvector_type_handle) : base_class(n, Type(), allocator(type_handle)) { } explicit pvector(size_type n, const Type &value, TypeHandle type_handle = pvector_type_handle) : base_class(n, value, allocator(type_handle)) { } pvector(const Type *begin, const Type *end, TypeHandle type_handle = pvector_type_handle) : base_class(begin, end, allocator(type_handle)) { } + +#ifdef USE_MOVE_SEMANTICS + pvector(pvector &&from) NOEXCEPT : base_class(move(from)) {}; + + pvector &operator =(pvector &&from) NOEXCEPT { + base_class::operator =(move(from)); + return *this; + } +#endif + + pvector &operator =(const pvector ©) { + base_class::operator =(copy); + return *this; + } }; #endif // USE_STL_ALLOCATOR From 753ae3974079ec54a49315381cecc091f25692ff Mon Sep 17 00:00:00 2001 From: rdb Date: Sun, 8 Apr 2018 13:50:33 +0200 Subject: [PATCH 11/17] readme: use MSVC 2015 instead of 2010 (see #288) [skip ci] --- README.md | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index fde01c38f0..cf45f0dfd2 100644 --- a/README.md +++ b/README.md @@ -43,24 +43,26 @@ Building Panda3D Windows ------- -We currently build using the Microsoft Visual C++ 2010 compiler. You do not -need Microsoft Visual Studio to build Panda3D, though - the relevant compilers -are included as part of the Windows 7.1 SDK. +We currently build using the Microsoft Visual C++ 2015 compiler. You will +also need to install the [Windows 10 SDK](https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk), +and if you intend to target Windows XP, you will also need the +[Windows 7.1 SDK](https://www.microsoft.com/en-us/download/details.aspx?id=8279). You will also need to have the third-party dependency libraries available for the build scripts to use. These are available from one of these two URLs, -depending on whether you are on a 32-bit or 64-bit system: -https://www.panda3d.org/download/panda3d-1.9.4/panda3d-1.9.4-tools-win32.zip -https://www.panda3d.org/download/panda3d-1.9.4/panda3d-1.9.4-tools-win64.zip +depending on whether you are on a 32-bit or 64-bit system, or you can +[click here](https://github.com/rdb/panda3d-thirdparty) for instructions on +building them from source. -(It is also possible to build using MSVC 2015 and 2017, which requires a -different set of thirdparty libraries, but that is not described here.) +http://rdb.name/thirdparty-vc14-x64.7z +http://rdb.name/thirdparty-vc14.7z After acquiring these dependencies, you may simply build Panda3D from the -command prompt using the following command: +command prompt using the following command. (Add the `--windows-sdk=10` +option if you don't need to support Windows XP.) ```bash -makepanda\makepanda.bat --everything --installer --no-eigen +makepanda\makepanda.bat --everything --installer --no-eigen --threads=2 ``` When the build succeeds, it will produce an .exe file that you can use to From 9ca0d089a669f3f3e63f58f329d354055917700b Mon Sep 17 00:00:00 2001 From: Younguk Kim Date: Tue, 10 Apr 2018 19:45:54 +0900 Subject: [PATCH 12/17] event: fix a crash when removing a task is not owned by user --- panda/src/event/asyncTaskChain.cxx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/panda/src/event/asyncTaskChain.cxx b/panda/src/event/asyncTaskChain.cxx index 47299d31ef..c7d732eca9 100644 --- a/panda/src/event/asyncTaskChain.cxx +++ b/panda/src/event/asyncTaskChain.cxx @@ -477,6 +477,7 @@ do_remove(AsyncTask *task, bool upon_death) { { int index = find_task_on_heap(_sleeping, task); nassertr(index != -1, false); + PT(AsyncTask) hold_task = task; _sleeping.erase(_sleeping.begin() + index); make_heap(_sleeping.begin(), _sleeping.end(), AsyncTaskSortWakeTime()); cleanup_task(task, upon_death, false); @@ -486,6 +487,7 @@ do_remove(AsyncTask *task, bool upon_death) { case AsyncTask::S_active: { // Active, but not being serviced, easy. + PT(AsyncTask) hold_task = task; int index = find_task_on_heap(_active, task); if (index != -1) { _active.erase(_active.begin() + index); @@ -769,7 +771,6 @@ cleanup_task(AsyncTask *task, bool upon_death, bool clean_exit) { } nassertv(task->_chain == this); - PT(AsyncTask) hold_task = task; task->_state = AsyncTask::S_inactive; task->_chain = nullptr; From 607af9ff4963db086b1233e195db45f7aeda68c1 Mon Sep 17 00:00:00 2001 From: rdb Date: Tue, 10 Apr 2018 15:29:43 +0200 Subject: [PATCH 13/17] flt: fix compile error on macOS 10.6 (which has no strnlen) --- pandatool/src/flt/fltBeadID.cxx | 4 ++-- pandatool/src/flt/fltRecord.cxx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pandatool/src/flt/fltBeadID.cxx b/pandatool/src/flt/fltBeadID.cxx index f55d3e88cc..32348d583e 100644 --- a/pandatool/src/flt/fltBeadID.cxx +++ b/pandatool/src/flt/fltBeadID.cxx @@ -78,8 +78,8 @@ extract_record(FltRecordReader &reader) { bool FltBeadID:: extract_ancillary(FltRecordReader &reader) { if (reader.get_opcode() == FO_long_id) { - vector_uchar s = reader.get_iterator().get_remaining_bytes(); - _id.assign((const char *)s.data(), strnlen((const char *)s.data(), s.size())); + DatagramIterator &di = reader.get_iterator(); + _id = di.get_fixed_string(di.get_remaining_size()); return true; } diff --git a/pandatool/src/flt/fltRecord.cxx b/pandatool/src/flt/fltRecord.cxx index a0af83e180..53844f0636 100644 --- a/pandatool/src/flt/fltRecord.cxx +++ b/pandatool/src/flt/fltRecord.cxx @@ -621,8 +621,8 @@ extract_record(FltRecordReader &) { bool FltRecord:: extract_ancillary(FltRecordReader &reader) { if (reader.get_opcode() == FO_comment) { - vector_uchar s = reader.get_iterator().get_remaining_bytes(); - _comment.assign((const char *)s.data(), strnlen((const char *)s.data(), s.size())); + DatagramIterator &di = reader.get_iterator(); + _comment = di.get_fixed_string(di.get_remaining_size()); return true; } From e1675f1a539c22fdb3c5ebdacaa6a5c0fb383152 Mon Sep 17 00:00:00 2001 From: rdb Date: Tue, 10 Apr 2018 15:31:48 +0200 Subject: [PATCH 14/17] makepanda: fix compile error on Windows with some versions of flex --- makepanda/makepanda.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/makepanda/makepanda.py b/makepanda/makepanda.py index b772a4a9e7..3f597f90b7 100755 --- a/makepanda/makepanda.py +++ b/makepanda/makepanda.py @@ -552,6 +552,7 @@ if (COMPILER == "MSVC"): PkgDisable("EGL") PkgDisable("CARBON") PkgDisable("COCOA") + DefSymbol("FLEX", "YY_NO_UNISTD_H") if (PkgSkip("PYTHON")==0): IncDirectory("ALWAYS", SDK["PYTHON"] + "/include") LibDirectory("ALWAYS", SDK["PYTHON"] + "/libs") @@ -1396,7 +1397,7 @@ def CompileBison(wobj, wsrc, opts): CopyFile(wdsth, GetOutputDir()+"/tmp/"+ifile+".h") # Finally, compile the generated source file. - CompileCxx(wobj,wdstc,opts) + CompileCxx(wobj, wdstc, opts + ["FLEX"]) ######################################################################## ## From 0cef19fd900977bd67eb7ed17200a62aa317f28c Mon Sep 17 00:00:00 2001 From: rdb Date: Wed, 11 Apr 2018 17:29:49 +0200 Subject: [PATCH 15/17] putil: more validation in DatagramInputFile::get_datagram Intended to fix test_file_corrupt for 32-bit platforms. See discussion in 89be2c19af74c62b57961469c779b324c69979f1 --- panda/src/putil/datagramInputFile.cxx | 28 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/panda/src/putil/datagramInputFile.cxx b/panda/src/putil/datagramInputFile.cxx index f3bd8a081e..f81381d02e 100644 --- a/panda/src/putil/datagramInputFile.cxx +++ b/panda/src/putil/datagramInputFile.cxx @@ -138,35 +138,45 @@ get_datagram(Datagram &data) { return true; } - streamsize num_bytes = (streamsize)num_bytes_32; + size_t num_bytes = (size_t)num_bytes_32; if (num_bytes_32 == (uint32_t)-1) { // Another special case for a value larger than 32 bits. - num_bytes = reader.get_uint64(); - } + uint64_t num_bytes_64 = reader.get_uint64(); - // Make sure we have a reasonable datagram size for putting into memory. - nassertr(num_bytes == (size_t)num_bytes, false); + if (_in->fail() || _in->eof()) { + _error = true; + return false; + } + + num_bytes = (size_t)num_bytes_64; + + // Make sure we have a reasonable datagram size for putting into memory. + if (num_bytes_64 != (uint64_t)num_bytes) { + _error = true; + return false; + } + } // Now, read the datagram itself. We construct an empty datagram, use // pad_bytes to make it big enough, and read *directly* into the datagram's // internal buffer. Doing this saves us a copy operation. data = Datagram(); - streamsize bytes_read = 0; + size_t bytes_read = 0; while (bytes_read < num_bytes) { - streamsize bytes_left = num_bytes - bytes_read; + size_t bytes_left = num_bytes - bytes_read; // Hold up a second - datagrams >4MB are pretty large by bam/network // standards. Let's take it 4MB at a time just in case the length is // corrupt, so we don't allocate potentially a few GBs of RAM only to // find a truncated file. - bytes_left = min(bytes_left, (streamsize)4*1024*1024); + bytes_left = min(bytes_left, (size_t)4*1024*1024); PTA_uchar buffer = data.modify_array(); buffer.resize(buffer.size() + bytes_left); unsigned char *ptr = &buffer.p()[bytes_read]; - _in->read((char *)ptr, bytes_left); + _in->read((char *)ptr, (streamsize)bytes_left); if (_in->fail() || _in->eof()) { _error = true; return false; From f979c4d3121ded6bc0ea3f2cfea68c5d686ab137 Mon Sep 17 00:00:00 2001 From: rdb Date: Wed, 11 Apr 2018 17:41:16 +0200 Subject: [PATCH 16/17] makepanda: don't use pkg-config for assimp It seems that they removed the -I option, which we need to build with. This commit is intended to fix the Ubuntu Bionic build. --- makepanda/makepanda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makepanda/makepanda.py b/makepanda/makepanda.py index 3f597f90b7..5ae064b649 100755 --- a/makepanda/makepanda.py +++ b/makepanda/makepanda.py @@ -828,7 +828,7 @@ if (COMPILER=="GCC"): SmartPkgEnable("EIGEN", "eigen3", (), ("Eigen/Dense",), target_pkg = 'ALWAYS') SmartPkgEnable("ARTOOLKIT", "", ("AR"), "AR/ar.h") SmartPkgEnable("FCOLLADA", "", ChooseLib(fcollada_libs, "FCOLLADA"), ("FCollada", "FCollada/FCollada.h")) - SmartPkgEnable("ASSIMP", "assimp", ("assimp"), "assimp") + SmartPkgEnable("ASSIMP", "", ("assimp"), "assimp") SmartPkgEnable("FFMPEG", ffmpeg_libs, ffmpeg_libs, ("libavformat/avformat.h", "libavcodec/avcodec.h", "libavutil/avutil.h")) SmartPkgEnable("SWSCALE", "libswscale", "libswscale", ("libswscale/swscale.h"), target_pkg = "FFMPEG", thirdparty_dir = "ffmpeg") SmartPkgEnable("SWRESAMPLE","libswresample", "libswresample", ("libswresample/swresample.h"), target_pkg = "FFMPEG", thirdparty_dir = "ffmpeg") From 4877e8350d58cafa42f12fe026c795656a4984a3 Mon Sep 17 00:00:00 2001 From: rdb Date: Wed, 18 Apr 2018 18:25:29 +0200 Subject: [PATCH 17/17] tests: fix unit test on 64-bit Ubuntu due to float imprecision [skip ci] --- tests/bullet/test_bullet_bam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/bullet/test_bullet_bam.py b/tests/bullet/test_bullet_bam.py index c6b9d2c282..11523a4f51 100644 --- a/tests/bullet/test_bullet_bam.py +++ b/tests/bullet/test_bullet_bam.py @@ -88,8 +88,8 @@ def test_minkowski_sum_shape(): assert type(shape) is type(shape2) assert shape.margin == shape2.margin assert shape.name == shape2.name - assert shape.transform_a.compare_to(shape2.transform_a, True) == 0 - assert shape.transform_b.compare_to(shape2.transform_b, True) == 0 + assert shape.transform_a.mat.compare_to(shape2.transform_a.mat) == 0 + assert shape.transform_b.mat.compare_to(shape2.transform_b.mat) == 0 assert type(shape.shape_a) == type(shape2.shape_a) assert type(shape.shape_b) == type(shape2.shape_b)