TheCloud/panda3d
1
0
Fork 0
mirror of https://github.com/panda3d/panda3d.git synced 2025-10-03 10:22:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

more fixes and optimizations to subfile patcher

Browse Source
This commit is contained in:
David Rose 2007-02-28 01:48:27 +00:00
parent 13c2658325
commit e51797e318
7 changed files with 451 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
panda/src/downloadertools/multify.cxx
View File

@ -32,8 +32,9 @@
bool create = false; // -c bool create = false; // -c
bool append = false; // -r bool append = false; // -r
bool update = false; // -u bool update = false; // -u
bool tlist = false; // -t bool tlist = false; // -t
bool extract = false; // -x bool extract = false; // -x
bool kill = false; // -k
bool verbose = false; // -v bool verbose = false; // -v
bool compress_flag = false; // -z bool compress_flag = false; // -z
int default_compression_level = 6; int default_compression_level = 6;
@ -138,9 +139,13 @@ help() {
" Extract the contents of an existing Multifile. The Subfiles named on\n" " Extract the contents of an existing Multifile. The Subfiles named on\n"
" the command line, or all Subfiles if nothing is named, are extracted\n" " the command line, or all Subfiles if nothing is named, are extracted\n"
" into the current directory or into whichever directory is specified\n" " into the current directory or into whichever directory is specified\n"
" with -C.\n\n\n" " with -C.\n\n"
" -k\n"
" Delete (kill) the named Subfiles from the Multifile. The Multifile\n"
" will be repacked after completion.\n\n"
"\n"
" You must always specify the following switch:\n\n" " You must always specify the following switch:\n\n"
" -f <multifile_name>\n" " -f <multifile_name>\n"
@ -437,6 +442,41 @@ extract_files(int argc, char *argv[]) {
return true; return true;
} }
bool
kill_files(int argc, char *argv[]) {
if (!multifile_name.exists()) {
cerr << multifile_name << " not found.\n";
return false;
}
PT(Multifile) multifile = new Multifile;
if (!multifile->open_read_write(multifile_name)) {
cerr << "Unable to open " << multifile_name << " for read/write.\n";
return false;
}
int i = 0;
while (i < multifile->get_num_subfiles()) {
string subfile_name = multifile->get_subfile_name(i);
if (is_named(subfile_name, argc, argv)) {
Filename filename = subfile_name;
if (verbose) {
cout << filename << "\n";
}
multifile->remove_subfile(i);
} else {
++i;
}
}
if (!multifile->repack()) {
cerr << "Failed to write " << multifile_name << ".\n";
return false;
}
return true;
}
const char * const char *
format_timestamp(bool record_timestamp, time_t timestamp) { format_timestamp(bool record_timestamp, time_t timestamp) {
static const size_t buffer_size = 512; static const size_t buffer_size = 512;
@ -582,7 +622,7 @@ main(int argc, char *argv[]) {
extern char *optarg; extern char *optarg;
extern int optind; extern int optind;
static const char *optflags = "crutxvz123456789Z:T:f:OC:ep:F:h"; static const char *optflags = "crutxkvz123456789Z:T:f:OC:ep:F:h";
int flag = getopt(argc, argv, optflags); int flag = getopt(argc, argv, optflags);
Filename rel_path; Filename rel_path;
while (flag != EOF) { while (flag != EOF) {
@ -602,6 +642,9 @@ main(int argc, char *argv[]) {
case 'x': case 'x':
extract = true; extract = true;
break; break;
case 'k':
kill = true;
break;
case 'v': case 'v':
verbose = true; verbose = true;
break; break;
@ -711,8 +754,8 @@ main(int argc, char *argv[]) {
argv += (optind - 1); argv += (optind - 1);
// We should have exactly one of these options. // We should have exactly one of these options.
if ((create?1:0) + (append?1:0) + (update?1:0) + (tlist?1:0) + (extract?1:0) != 1) { if ((create?1:0) + (append?1:0) + (update?1:0) + (tlist?1:0) + (extract?1:0) + (kill?1:0) != 1) {
cerr << "Exactly one of -c, -r, -u, -t, -x must be specified.\n"; cerr << "Exactly one of -c, -r, -u, -t, -x, -k must be specified.\n";
usage(); usage();
return 1; return 1;
} }
@ -734,6 +777,11 @@ main(int argc, char *argv[]) {
cerr << "Warning: -T ignored on extract.\n"; cerr << "Warning: -T ignored on extract.\n";
} }
okflag = extract_files(argc, argv); okflag = extract_files(argc, argv);
} else if (kill) {
if (got_record_timestamp_flag) {
cerr << "Warning: -T ignored on kill.\n";
}
okflag = kill_files(argc, argv);
} else { // list } else { // list
if (got_record_timestamp_flag) { if (got_record_timestamp_flag) {
cerr << "Warning: -T ignored on list.\n"; cerr << "Warning: -T ignored on list.\n";

38
panda/src/express/hashVal.cxx
View File

@ -211,6 +211,25 @@ hash_file(const Filename &filename) {
return false; return false;
} }
bool result = hash_stream(*istr);
vfs->close_read_file(istr);
return result;
}
#endif // HAVE_OPENSSL
#ifdef HAVE_OPENSSL
////////////////////////////////////////////////////////////////////
// Function: HashVal::hash_stream
// Access: Published
// Description: Generates the hash value from the indicated file.
// Returns true on success, false if the file cannot be
// read. This method is only defined if we have the
// OpenSSL library (which provides md5 functionality)
// available.
////////////////////////////////////////////////////////////////////
bool HashVal::
hash_stream(istream &stream) {
unsigned char md[16]; unsigned char md[16];
MD5_CTX ctx; MD5_CTX ctx;
@ -219,15 +238,21 @@ hash_file(const Filename &filename) {
static const int buffer_size = 1024; static const int buffer_size = 1024;
char buffer[buffer_size]; char buffer[buffer_size];
istr->read(buffer, buffer_size); // Seek the stream to the beginning in case it wasn't there already.
size_t count = istr->gcount(); stream.seekg(0, ios::beg);
stream.read(buffer, buffer_size);
size_t count = stream.gcount();
while (count != 0) { while (count != 0) {
MD5_Update(&ctx, buffer, count); MD5_Update(&ctx, buffer, count);
istr->read(buffer, buffer_size); stream.read(buffer, buffer_size);
count = istr->gcount(); count = stream.gcount();
} }
// Clear the fail bit so the caller can still read the stream (if it
// wants to).
stream.clear();
vfs->close_read_file(istr);
MD5_Final(md, &ctx); MD5_Final(md, &ctx);
// Store the individual bytes as big-endian ints, from historical // Store the individual bytes as big-endian ints, from historical
@ -239,7 +264,10 @@ hash_file(const Filename &filename) {
return true; return true;
} }
#endif // HAVE_OPENSSL
#ifdef HAVE_OPENSSL
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
// Function: HashVal::hash_buffer // Function: HashVal::hash_buffer
// Access: Published // Access: Published

1
panda/src/express/hashVal.h
View File

@ -71,6 +71,7 @@ PUBLISHED:
#ifdef HAVE_OPENSSL #ifdef HAVE_OPENSSL
bool hash_file(const Filename &filename); bool hash_file(const Filename &filename);
bool hash_stream(istream &stream);
INLINE void hash_ramfile(const Ramfile &ramfile); INLINE void hash_ramfile(const Ramfile &ramfile);
INLINE void hash_string(const string &data); INLINE void hash_string(const string &data);
void hash_buffer(const char *buffer, int length); void hash_buffer(const char *buffer, int length);

5
panda/src/express/multifile.cxx
View File

@ -1161,6 +1161,7 @@ open_read(istream *multifile_stream) {
_timestamp = time(NULL); _timestamp = time(NULL);
_timestamp_dirty = true; _timestamp_dirty = true;
_read = multifile_stream; _read = multifile_stream;
_read->seekg(0, ios::beg);
return read_index(); return read_index();
} }
@ -1180,6 +1181,7 @@ open_write(ostream *multifile_stream) {
_timestamp = time(NULL); _timestamp = time(NULL);
_timestamp_dirty = true; _timestamp_dirty = true;
_write = multifile_stream; _write = multifile_stream;
_write->seekp(0, ios::beg);
return true; return true;
} }
@ -1201,6 +1203,7 @@ open_read_write(iostream *multifile_stream) {
_timestamp_dirty = true; _timestamp_dirty = true;
_read = multifile_stream; _read = multifile_stream;
_write = multifile_stream; _write = multifile_stream;
_write->seekp(0, ios::beg);
// Check whether the read stream is empty. // Check whether the read stream is empty.
_read->seekg(0, ios::end); _read->seekg(0, ios::end);
@ -1211,7 +1214,7 @@ open_read_write(iostream *multifile_stream) {
// The read stream is not empty, so we'd better have a valid // The read stream is not empty, so we'd better have a valid
// Multifile. // Multifile.
_read->seekg(0); _read->seekg(0, ios::beg);
return read_index(); return read_index();
} }

517
panda/src/express/patchfile.cxx
View File

@ -26,6 +26,7 @@
#include "streamReader.h" #include "streamReader.h"
#include "streamWriter.h" #include "streamWriter.h"
#include "multifile.h" #include "multifile.h"
#include "hashVal.h"
#include <stdio.h> // for tempnam #include <stdio.h> // for tempnam
@ -109,6 +110,7 @@ const PN_uint32 Patchfile::_HASH_MASK = (PN_uint32(1) << Patchfile::_HASH_BITS)
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
Patchfile:: Patchfile::
Patchfile() { Patchfile() {
_hash_table = NULL;
PT(Buffer) buffer = new Buffer(patchfile_buffer_size); PT(Buffer) buffer = new Buffer(patchfile_buffer_size);
init(buffer); init(buffer);
} }
@ -120,6 +122,7 @@ Patchfile() {
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
Patchfile:: Patchfile::
Patchfile(PT(Buffer) buffer) { Patchfile(PT(Buffer) buffer) {
_hash_table = NULL;
init(buffer); init(buffer);
} }
@ -146,6 +149,10 @@ init(PT(Buffer) buffer) {
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
Patchfile:: Patchfile::
~Patchfile() { ~Patchfile() {
if (_hash_table != (PN_uint32 *)NULL) {
delete[] _hash_table;
}
if (true == _initiated) if (true == _initiated)
cleanup(); cleanup();
} }
@ -299,7 +306,7 @@ run() {
_total_bytes_processed += (int)ADD_length; _total_bytes_processed += (int)ADD_length;
// if there are bytes to add, read them from patch file and write them to output // if there are bytes to add, read them from patch file and write them to output
if (express_cat.is_spam()) { if (express_cat.is_spam() && ADD_length != 0) {
express_cat.spam() express_cat.spam()
<< "ADD: " << ADD_length << " (to " << "ADD: " << ADD_length << " (to "
<< _write_stream.tellp() << ")" << endl; << _write_stream.tellp() << ")" << endl;
@ -762,14 +769,12 @@ find_longest_match(PN_uint32 new_pos, PN_uint32 &copy_pos, PN_uint16 &copy_lengt
// Description: // Description:
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
void Patchfile:: void Patchfile::
emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer, emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer) {
PN_uint32 ADD_pos) {
nassertv(length == (PN_uint16)length); //we only write a uint16 nassertv(length == (PN_uint16)length); //we only write a uint16
if (express_cat.is_spam()) { if (express_cat.is_spam()) {
express_cat.spam() express_cat.spam()
<< "ADD: " << length << " (to " << ADD_pos << ")" << endl; << "ADD: " << length << " (to " << _add_pos << ")" << endl;
} }
// write ADD length // write ADD length
@ -780,6 +785,8 @@ emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer,
if (length > 0) { if (length > 0) {
patch_writer.append_data(buffer, (PN_uint16)length); patch_writer.append_data(buffer, (PN_uint16)length);
} }
_add_pos += length;
} }
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
@ -787,29 +794,28 @@ emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer,
// Access: Private // Access: Private
// Description: // Description:
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
PN_uint32 Patchfile:: void Patchfile::
emit_COPY(ostream &write_stream, PN_uint32 length, PN_uint32 COPY_pos, emit_COPY(ostream &write_stream, PN_uint32 length, PN_uint32 copy_pos) {
PN_uint32 last_copy_pos, PN_uint32 ADD_pos) { nassertv(length == (PN_uint16)length); //we only write a uint16
nassertr(length == (PN_uint16)length, last_copy_pos); //we only write a uint16 PN_int32 offset = (int)copy_pos - (int)_last_copy_pos;
PN_int32 offset = (int)COPY_pos - (int)last_copy_pos;
if (express_cat.is_spam()) { if (express_cat.is_spam()) {
express_cat.spam() express_cat.spam()
<< "COPY: " << length << " bytes from offset " << offset << "COPY: " << length << " bytes from offset " << offset
<< " (from " << COPY_pos << " to " << ADD_pos << ")" << endl; << " (from " << copy_pos << " to " << _add_pos << ")" << endl;
} }
// write COPY length // write COPY length
StreamWriter patch_writer(write_stream); StreamWriter patch_writer(write_stream);
patch_writer.add_uint16((PN_uint16)length); patch_writer.add_uint16((PN_uint16)length);
if((PN_uint16)length > 0) { if ((PN_uint16)length != 0) {
// write COPY offset // write COPY offset
patch_writer.add_int32(offset); patch_writer.add_int32(offset);
_last_copy_pos = copy_pos + length;
} }
return COPY_pos + length; _add_pos += length;
} }
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
@ -819,42 +825,91 @@ emit_COPY(ostream &write_stream, PN_uint32 length, PN_uint32 COPY_pos,
// pair as needed to work around the 16-bit chunk size // pair as needed to work around the 16-bit chunk size
// limit. // limit.
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
PN_uint32 Patchfile:: void Patchfile::
emit_add_and_copy(ostream &write_stream, emit_add_and_copy(ostream &write_stream,
PN_uint32 add_length, const char *add_buffer, PN_uint32 add_length, const char *add_buffer,
PN_uint32 copy_length, PN_uint32 copy_pos, PN_uint32 last_copy_pos, PN_uint32 copy_length, PN_uint32 copy_pos) {
PN_uint32 add_pos) { if (add_length == 0 && copy_length == 0) {
while (add_length > 65535) { // Don't accidentally emit a termination code.
return;
}
static const PN_uint16 max_write = 65535;
while (add_length > max_write) {
// Overflow. This chunk is too large to fit into a single // Overflow. This chunk is too large to fit into a single
// ADD block, so we have to write it as multiple ADDs. // ADD block, so we have to write it as multiple ADDs.
static const PN_uint16 max_write = 65535; emit_ADD(write_stream, max_write, add_buffer);
emit_ADD(write_stream, max_write, add_buffer, add_pos);
add_pos += max_write;
add_buffer += max_write; add_buffer += max_write;
add_length -= max_write; add_length -= max_write;
emit_COPY(write_stream, 0, last_copy_pos, last_copy_pos, add_pos); emit_COPY(write_stream, 0, 0);
} }
emit_ADD(write_stream, add_length, add_buffer, add_pos); emit_ADD(write_stream, add_length, add_buffer);
while (copy_length > 65535) { while (copy_length > max_write) {
// Overflow. // Overflow.
static const PN_uint16 max_write = 65535; emit_COPY(write_stream, max_write, copy_pos);
last_copy_pos =
emit_COPY(write_stream, max_write, copy_pos, last_copy_pos, add_pos);
copy_pos += max_write; copy_pos += max_write;
add_pos += max_write;
copy_length -= max_write; copy_length -= max_write;
emit_ADD(write_stream, 0, NULL, add_pos); emit_ADD(write_stream, 0, NULL);
} }
last_copy_pos = emit_COPY(write_stream, copy_length, copy_pos);
emit_COPY(write_stream, copy_length, copy_pos, last_copy_pos, add_pos);
return last_copy_pos;
} }
////////////////////////////////////////////////////////////////////
// Function: Patchfile::cache_add_and_copy
// Access: Private
// Description: Potentially emits one or more add/copy pairs. The
// current state is saved, so as to minimize wasted
// emits from consecutive adds or copies.
////////////////////////////////////////////////////////////////////
void Patchfile::
cache_add_and_copy(ostream &write_stream,
PN_uint32 add_length, const char *add_buffer,
PN_uint32 copy_length, PN_uint32 copy_pos) {
if (add_length != 0) {
if (_cache_copy_length != 0) {
// Have to flush.
cache_flush(write_stream);
}
// Add the string to the current cache.
_cache_add_data += string(add_buffer, add_length);
}
if (copy_length != 0) {
if (_cache_copy_length == 0) {
// Start a new copy phase.
_cache_copy_start = copy_pos;
_cache_copy_length = copy_length;
} else if (_cache_copy_start + _cache_copy_length == copy_pos) {
// We can just tack on the copy to what we've already got.
_cache_copy_length += copy_length;
} else {
// It's a discontinuous copy. We have to flush.
cache_flush(write_stream);
_cache_copy_start = copy_pos;
_cache_copy_length = copy_length;
}
}
}
////////////////////////////////////////////////////////////////////
// Function: Patchfile::cache_flush
// Access: Private
// Description: Closes any copy or add phases that are still open
// after a previous call to cache_add_and_copy().
////////////////////////////////////////////////////////////////////
void Patchfile::
cache_flush(ostream &write_stream) {
emit_add_and_copy(write_stream,
_cache_add_data.size(), _cache_add_data.data(),
_cache_copy_length, _cache_copy_start);
_cache_add_data = string();
_cache_copy_length = 0;
}
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
@ -865,8 +920,7 @@ emit_add_and_copy(ostream &write_stream,
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
void Patchfile:: void Patchfile::
write_header(ostream &write_stream, write_header(ostream &write_stream,
char *buffer_orig, PN_uint32 source_file_length, istream &stream_orig, istream &stream_new) {
char *buffer_new, PN_uint32 result_file_length) {
// prepare to write the patch file header // prepare to write the patch file header
START_PROFILE(writeHeader); START_PROFILE(writeHeader);
@ -875,19 +929,33 @@ write_header(ostream &write_stream,
StreamWriter patch_writer(write_stream); StreamWriter patch_writer(write_stream);
patch_writer.add_uint32(_magic_number); patch_writer.add_uint32(_magic_number);
patch_writer.add_uint16(_current_version); patch_writer.add_uint16(_current_version);
patch_writer.add_uint32(source_file_length);
{ stream_orig.seekg(0, ios::end);
// calc MD5 of original file streampos source_file_length = stream_orig.tellg();
_MD5_ofSource.hash_buffer(buffer_orig, source_file_length); patch_writer.add_uint32((PN_uint32)source_file_length);
// add it to the header
_MD5_ofSource.write_stream(patch_writer); // calc MD5 of original file
_MD5_ofSource.hash_stream(stream_orig);
// add it to the header
_MD5_ofSource.write_stream(patch_writer);
if (express_cat.is_debug()) {
express_cat.debug()
<< "Orig: " << _MD5_ofSource << "\n";
} }
patch_writer.add_uint32(result_file_length);
{ stream_new.seekg(0, ios::end);
// calc MD5 of resultant patched file streampos result_file_length = stream_new.tellg();
_MD5_ofResult.hash_buffer(buffer_new, result_file_length); patch_writer.add_uint32((PN_uint32)result_file_length);
// add it to the header
_MD5_ofResult.write_stream(patch_writer); // calc MD5 of resultant patched file
_MD5_ofResult.hash_stream(stream_new);
// add it to the header
_MD5_ofResult.write_stream(patch_writer);
if (express_cat.is_debug()) {
express_cat.debug()
<< " New: " << _MD5_ofResult << "\n";
} }
END_PROFILE(writeHeader, "writing patch file header"); END_PROFILE(writeHeader, "writing patch file header");
@ -900,11 +968,11 @@ write_header(ostream &write_stream,
// Writes the patchfile terminator. // Writes the patchfile terminator.
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
void Patchfile:: void Patchfile::
write_terminator(ostream &write_stream, PN_uint32 result_file_length, write_terminator(ostream &write_stream) {
PN_uint32 last_copy_pos) { cache_flush(write_stream);
// write terminator (null ADD, null COPY) // write terminator (null ADD, null COPY)
emit_ADD(write_stream, 0, NULL, result_file_length); emit_ADD(write_stream, 0, NULL);
emit_COPY(write_stream, 0, last_copy_pos, last_copy_pos, result_file_length); emit_COPY(write_stream, 0, 0);
} }
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
@ -914,24 +982,48 @@ write_terminator(ostream &write_stream, PN_uint32 result_file_length,
// Computes the patches for the entire file (if it is // Computes the patches for the entire file (if it is
// not a multifile) or for a single subfile (if it is) // not a multifile) or for a single subfile (if it is)
// //
// Returns last_copy_pos, the last byte position from // Returns true if successful, false on error.
// which we copied from the original file.
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
PN_uint32 Patchfile:: bool Patchfile::
compute_patches(ostream &write_stream, PN_uint32 last_copy_pos, compute_patches(ostream &write_stream, PN_uint32 copy_offset,
PN_uint32 copy_offset, istream &stream_orig, istream &stream_new) {
char *buffer_orig, PN_uint32 source_file_length, // read in original file
char *buffer_new, PN_uint32 result_file_length) { stream_orig.seekg(0, ios::end);
nassertr(stream_orig, false);
PN_uint32 source_file_length = stream_orig.tellg();
if (express_cat.is_debug()) {
express_cat.debug()
<< "Allocating " << source_file_length << " bytes to read orig\n";
}
char *buffer_orig = new char[source_file_length];
stream_orig.seekg(0, ios::beg);
stream_orig.read(buffer_orig, source_file_length);
// read in new file
stream_new.seekg(0, ios::end);
PN_uint32 result_file_length = stream_new.tellg();
nassertr(stream_new, false);
if (express_cat.is_debug()) {
express_cat.debug()
<< "Allocating " << result_file_length << " bytes to read new\n";
}
char *buffer_new = new char[result_file_length];
stream_new.seekg(0, ios::beg);
stream_new.read(buffer_new, result_file_length);
START_PROFILE(allocTables); START_PROFILE(allocTables);
// allocate hash/link tables // allocate hash/link tables
if (express_cat.is_debug()) { if (_hash_table == (PN_uint32 *)NULL) {
express_cat.debug() if (express_cat.is_debug()) {
<< "Allocating hashtable of size " << _HASHTABLESIZE << " * 4\n"; express_cat.debug()
<< "Allocating hashtable of size " << _HASHTABLESIZE << " * 4\n";
}
_hash_table = new PN_uint32[_HASHTABLESIZE];
} }
PN_uint32* hash_table = new PN_uint32[_HASHTABLESIZE];
if (express_cat.is_debug()) { if (express_cat.is_debug()) {
express_cat.debug() express_cat.debug()
<< "Allocating linktable of size " << source_file_length << " * 4\n"; << "Allocating linktable of size " << source_file_length << " * 4\n";
@ -944,7 +1036,7 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
START_PROFILE(buildTables); START_PROFILE(buildTables);
// build hash and link tables for original file // build hash and link tables for original file
build_hash_link_tables(buffer_orig, source_file_length, hash_table, link_table); build_hash_link_tables(buffer_orig, source_file_length, _hash_table, link_table);
END_PROFILE(buildTables, "building hash and link tables"); END_PROFILE(buildTables, "building hash and link tables");
@ -952,7 +1044,7 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
START_PROFILE(buildPatchfile); START_PROFILE(buildPatchfile);
PN_uint32 new_pos = 0; PN_uint32 new_pos = 0;
PN_uint32 ADD_pos = new_pos; // this is the position for the start of ADD operations PN_uint32 start_pos = new_pos; // this is the position for the start of ADD operations
if(((PN_uint32) result_file_length) >= _footprint_length) if(((PN_uint32) result_file_length) >= _footprint_length)
{ {
@ -962,7 +1054,7 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
PN_uint32 COPY_pos; PN_uint32 COPY_pos;
PN_uint16 COPY_length; PN_uint16 COPY_length;
find_longest_match(new_pos, COPY_pos, COPY_length, hash_table, link_table, find_longest_match(new_pos, COPY_pos, COPY_length, _hash_table, link_table,
buffer_orig, source_file_length, buffer_new, result_file_length); buffer_orig, source_file_length, buffer_new, result_file_length);
// if no match or match not longer than footprint length, skip to next byte // if no match or match not longer than footprint length, skip to next byte
@ -971,22 +1063,16 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
new_pos++; new_pos++;
} else { } else {
// emit ADD for all skipped bytes // emit ADD for all skipped bytes
int num_skipped = (int)new_pos - (int)ADD_pos; int num_skipped = (int)new_pos - (int)start_pos;
if (express_cat.is_spam()) { if (express_cat.is_spam()) {
express_cat.spam() express_cat.spam()
<< "build: num_skipped = " << num_skipped << "build: num_skipped = " << num_skipped
<< endl; << endl;
} }
last_copy_pos = cache_add_and_copy(write_stream, num_skipped, &buffer_new[start_pos],
emit_add_and_copy(write_stream, num_skipped, &buffer_new[ADD_pos], COPY_length, COPY_pos + copy_offset);
COPY_length, COPY_pos + copy_offset,
last_copy_pos, ADD_pos);
ADD_pos += num_skipped;
nassertr(ADD_pos == new_pos, last_copy_pos);
// skip past match in new_file
new_pos += (PN_uint32)COPY_length; new_pos += (PN_uint32)COPY_length;
ADD_pos = new_pos; start_pos = new_pos;
} }
} }
} }
@ -994,28 +1080,28 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
if (express_cat.is_spam()) { if (express_cat.is_spam()) {
express_cat.spam() express_cat.spam()
<< "build: result_file_length = " << result_file_length << "build: result_file_length = " << result_file_length
<< " ADD_pos = " << ADD_pos << " start_pos = " << start_pos
<< endl; << endl;
} }
// are there still more bytes left in the new file? // are there still more bytes left in the new file?
if (ADD_pos != result_file_length) { if (start_pos != result_file_length) {
// emit ADD for all remaining bytes // emit ADD for all remaining bytes
PN_uint32 remaining_bytes = result_file_length - ADD_pos; PN_uint32 remaining_bytes = result_file_length - start_pos;
last_copy_pos = cache_add_and_copy(write_stream, remaining_bytes, &buffer_new[start_pos],
emit_add_and_copy(write_stream, remaining_bytes, &buffer_new[ADD_pos], 0, 0);
0, last_copy_pos, last_copy_pos, ADD_pos); start_pos += remaining_bytes;
ADD_pos += remaining_bytes;
nassertr(ADD_pos == result_file_length, last_copy_pos);
} }
END_PROFILE(buildPatchfile, "building patch file"); END_PROFILE(buildPatchfile, "building patch file");
delete[] hash_table;
delete[] link_table; delete[] link_table;
return last_copy_pos; delete[] buffer_orig;
delete[] buffer_new;
return true;
} }
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
@ -1028,36 +1114,33 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
// much faster for large Multifiles that contain many // much faster for large Multifiles that contain many
// small subfiles. // small subfiles.
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
PN_uint32 Patchfile:: bool Patchfile::
compute_mf_patches(ostream &write_stream, compute_mf_patches(ostream &write_stream,
char *buffer_orig, PN_uint32 source_file_length, istream &stream_orig, istream &stream_new) {
char *buffer_new, PN_uint32 result_file_length) {
string string_orig(buffer_orig, source_file_length);
string string_new(buffer_new, result_file_length);
istringstream strm_orig(string_orig);
istringstream strm_new(string_new);
Multifile mf_orig, mf_new; Multifile mf_orig, mf_new;
if (!mf_orig.open_read(&strm_orig) || if (!mf_orig.open_read(&stream_orig) ||
!mf_new.open_read(&strm_new)) { !mf_new.open_read(&stream_new)) {
express_cat.error() express_cat.error()
<< "Input multifiles appear to be corrupt.\n"; << "Input multifiles appear to be corrupt.\n";
return 0; return false;
} }
if (mf_new.needs_repack()) { if (mf_new.needs_repack()) {
express_cat.error() express_cat.error()
<< "Input multifiles need to be repacked.\n"; << "Input multifiles need to be repacked.\n";
return 0; return false;
} }
// First, compute the patch for the header / index. // First, compute the patch for the header / index.
PN_uint32 last_copy_pos = {
compute_patches(write_stream, 0, 0, ISubStream index_orig(&stream_orig, 0, mf_orig.get_index_end());
buffer_orig, (PN_uint32)mf_orig.get_index_end(), ISubStream index_new(&stream_new, 0, mf_new.get_index_end());
buffer_new, (PN_uint32)mf_new.get_index_end()); if (!compute_patches(write_stream, 0, index_orig, index_new)) {
PN_uint32 add_pos = (PN_uint32)mf_new.get_index_end(); return false;
}
nassertr(_add_pos + _cache_add_data.size() + _cache_copy_length == mf_new.get_index_end(), false);
}
// Now walk through each subfile in the new multifile. If a // Now walk through each subfile in the new multifile. If a
// particular subfile exists in both source files, we compute the // particular subfile exists in both source files, we compute the
@ -1066,54 +1149,128 @@ compute_mf_patches(ostream &write_stream,
// never even notice this case). // never even notice this case).
int new_num_subfiles = mf_new.get_num_subfiles(); int new_num_subfiles = mf_new.get_num_subfiles();
for (int ni = 0; ni < new_num_subfiles; ++ni) { for (int ni = 0; ni < new_num_subfiles; ++ni) {
nassertr(add_pos == mf_new.get_subfile_internal_start(ni), last_copy_pos); nassertr(_add_pos + _cache_add_data.size() + _cache_copy_length == mf_new.get_subfile_internal_start(ni), false);
string name = mf_new.get_subfile_name(ni); string name = mf_new.get_subfile_name(ni);
int oi = mf_orig.find_subfile(name); int oi = mf_orig.find_subfile(name);
/*
if (oi < 0) { if (oi < 0) {
// This subfile exists in the new file, but not in the original string new_ext = Filename(name).get_extension();
// file. Trivially add it. if (new_ext != "jpg") {
// This is a newly-added subfile. Look for another subfile with
// the same extension, so we can generate a patch against that
// other subfile--the idea is that there are likely to be
// similar byte sequences in files with the same extension, so
// "patching" a new subfile against a different subfile may come
// out smaller than baldly adding the new subfile.
size_t new_size = mf_new.get_subfile_internal_length(ni);
int orig_num_subfiles = mf_orig.get_num_subfiles();
size_t best_size = 0;
for (int i = 0; i < orig_num_subfiles; ++i) {
string orig_ext = Filename(mf_orig.get_subfile_name(i)).get_extension();
if (orig_ext == new_ext) {
size_t orig_size = mf_orig.get_subfile_internal_length(i);
// Find the smallest candidate that is no smaller than our
// target file. If all the candidates are smaller than our
// target file, choose the largest of them.
if ((best_size < new_size && orig_size > best_size) ||
(best_size >= new_size && orig_size >= new_size && orig_size < best_size)) {
best_size = orig_size;
oi = i;
}
}
}
}
}
*/
if (oi < 0) {
// This is a newly-added subfile, and we didn't find another
// subfile with a matching extension. Add it the hard way.
express_cat.info() express_cat.info()
<< "Adding subfile " << mf_new.get_subfile_name(ni) << "\n"; << "Adding subfile " << mf_new.get_subfile_name(ni) << "\n";
PN_uint32 new_start = (PN_uint32)mf_new.get_subfile_internal_start(ni);
PN_uint32 new_size = (PN_uint32)mf_new.get_subfile_internal_length(ni); streampos new_start = mf_new.get_subfile_internal_start(ni);
last_copy_pos = size_t new_size = mf_new.get_subfile_internal_length(ni);
emit_add_and_copy(write_stream, new_size, &buffer_new[new_start], char *buffer_new = new char[new_size];
0, last_copy_pos, last_copy_pos, add_pos); stream_new.seekg(new_start, ios::beg);
add_pos += new_size; stream_new.read(buffer_new, new_size);
cache_add_and_copy(write_stream, new_size, buffer_new, 0, 0);
delete[] buffer_new;
} else { } else {
// This subfile exists in both the original and the new files. // This subfile exists in both the original and the new files.
// Patch it. // Patch it.
PN_uint32 orig_start = (PN_uint32)mf_orig.get_subfile_internal_start(oi);
PN_uint32 orig_size = (PN_uint32)mf_orig.get_subfile_internal_length(oi); streampos orig_start = mf_orig.get_subfile_internal_start(oi);
PN_uint32 new_start = (PN_uint32)mf_new.get_subfile_internal_start(ni); size_t orig_size = mf_orig.get_subfile_internal_length(oi);
PN_uint32 new_size = (PN_uint32)mf_new.get_subfile_internal_length(ni); ISubStream subfile_orig(&stream_orig, orig_start, orig_start + (streampos)orig_size);
if (orig_size == new_size &&
memcmp(&buffer_orig[orig_start], &buffer_new[new_start], new_size) == 0) { streampos new_start = mf_new.get_subfile_internal_start(ni);
// Actually, the subfile is unchanged; just emit it. size_t new_size = mf_new.get_subfile_internal_length(ni);
ISubStream subfile_new(&stream_new, new_start, new_start + (streampos)new_size);
bool is_unchanged = false;
if (orig_size == new_size) {
HashVal hash_orig, hash_new;
hash_orig.hash_stream(subfile_orig);
hash_new.hash_stream(subfile_new);
if (hash_orig == hash_new) {
// Actually, the subfile is unchanged; just emit it.
is_unchanged = true;
}
}
if (is_unchanged) {
if (express_cat.is_debug()) { if (express_cat.is_debug()) {
express_cat.debug() express_cat.debug()
<< "Keeping subfile " << mf_new.get_subfile_name(ni) << "\n"; << "Keeping subfile " << mf_new.get_subfile_name(ni);
if (mf_orig.get_subfile_name(oi) != mf_new.get_subfile_name(ni)) {
express_cat.debug(false)
<< " (identical to " << mf_orig.get_subfile_name(oi) << ")";
}
express_cat.debug(false) << "\n";
} }
last_copy_pos = cache_add_and_copy(write_stream, 0, NULL,
emit_add_and_copy(write_stream, 0, NULL, orig_size, orig_start);
new_size, orig_start, last_copy_pos,
add_pos);
} else { } else {
express_cat.info() express_cat.info()
<< "Patching subfile " << mf_new.get_subfile_name(ni) << "\n"; << "Patching subfile " << mf_new.get_subfile_name(ni);
last_copy_pos = if (mf_orig.get_subfile_name(oi) != mf_new.get_subfile_name(ni)) {
compute_patches(write_stream, last_copy_pos, orig_start, express_cat.info(false)
&buffer_orig[orig_start], orig_size, << " (against " << mf_orig.get_subfile_name(oi) << ")";
&buffer_new[new_start], new_size); }
express_cat.info(false) << "\n";
if (!compute_patches(write_stream, orig_start,
subfile_orig, subfile_new)) {
return false;
}
/*
// Simply copy the new file; don't attempt to patch.
if (express_cat.is_debug()) {
express_cat.debug()
<< "Copying subfile " << mf_new.get_subfile_name(ni) << "\n";
}
streampos new_start = mf_new.get_subfile_internal_start(ni);
size_t new_size = mf_new.get_subfile_internal_length(ni);
char *buffer_new = new char[new_size];
stream_new.seekg(new_start, ios::beg);
stream_new.read(buffer_new, new_size);
cache_add_and_copy(write_stream, new_size, buffer_new,
0, 0);
delete[] buffer_new;
*/
} }
add_pos += new_size;
} }
} }
nassertr(add_pos == result_file_length, last_copy_pos); return true;
return last_copy_pos;
} }
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
@ -1134,8 +1291,6 @@ build(Filename file_orig, Filename file_new, Filename patch_name) {
START_PROFILE(overall); START_PROFILE(overall);
START_PROFILE(readFiles);
// Open the original file for read // Open the original file for read
ifstream stream_orig; ifstream stream_orig;
file_orig.set_binary(); file_orig.set_binary();
@ -1162,73 +1317,67 @@ build(Filename file_orig, Filename file_new, Filename patch_name) {
return false; return false;
} }
// read in original file _last_copy_pos = 0;
stream_orig.seekg(0, ios::end); _add_pos = 0;
PN_uint32 source_file_length = stream_orig.tellg(); _cache_add_data = string();
if (express_cat.is_debug()) { _cache_copy_start = 0;
express_cat.debug() _cache_copy_length = 0;
<< "Allocating " << source_file_length << " bytes to read "
<< file_orig << "\n";
}
char *buffer_orig = new char[source_file_length]; write_header(write_stream, stream_orig, stream_new);
stream_orig.seekg(0, ios::beg);
stream_orig.read(buffer_orig, source_file_length);
// read in new file
stream_new.seekg(0, ios::end);
PN_uint32 result_file_length = stream_new.tellg();
if (express_cat.is_debug()) {
express_cat.debug()
<< "Allocating " << result_file_length << " bytes to write "
<< file_new << "\n";
}
char *buffer_new = new char[result_file_length];
stream_new.seekg(0, ios::beg);
stream_new.read(buffer_new, result_file_length);
// close the original and new files (we have em in memory)
stream_orig.close();
stream_new.close();
END_PROFILE(readFiles, "reading files");
write_header(write_stream, buffer_orig, source_file_length,
buffer_new, result_file_length);
PN_uint32 last_copy_pos;
// Check whether our input files are Panda multifiles. // Check whether our input files are Panda multifiles.
bool is_multifile = false; bool is_multifile = false;
if (_allow_multifile) { if (_allow_multifile) {
if (file_orig.get_extension() == "mf" || file_new.get_extension() == "mf") { if (file_orig.get_extension() == "mf" || file_new.get_extension() == "mf") {
// Read the first n bytes of both files for the Multifile magic
// number.
string magic_number = Multifile::get_magic_number(); string magic_number = Multifile::get_magic_number();
if (source_file_length > magic_number.size() && char *buffer = new char[magic_number.size()];
result_file_length > magic_number.size() && stream_orig.seekg(0, ios::beg);
memcmp(buffer_orig, magic_number.data(), magic_number.size()) == 0 && stream_orig.read(buffer, magic_number.size());
memcmp(buffer_new, magic_number.data(), magic_number.size()) == 0) {
is_multifile = true; if (stream_orig.gcount() == magic_number.size() &&
memcmp(buffer, magic_number.data(), magic_number.size()) == 0) {
stream_new.seekg(0, ios::beg);
stream_new.read(buffer, magic_number.size());
if (stream_new.gcount() == magic_number.size() &&
memcmp(buffer, magic_number.data(), magic_number.size()) == 0) {
is_multifile = true;
}
} }
delete[] buffer;
} }
} }
if (is_multifile) { if (is_multifile) {
last_copy_pos = if (express_cat.is_debug()) {
compute_mf_patches(write_stream, buffer_orig, source_file_length, express_cat.debug()
buffer_new, result_file_length); << "Input files appear to be Panda Multifiles.\n";
}
if (!compute_mf_patches(write_stream, stream_orig, stream_new)) {
return false;
}
} else { } else {
last_copy_pos = if (express_cat.is_debug()) {
compute_patches(write_stream, 0, 0, express_cat.debug()
buffer_orig, source_file_length, << "Input files are NOT Panda Multifiles.\n";
buffer_new, result_file_length); }
if (!compute_patches(write_stream, 0, stream_orig, stream_new)) {
return false;
}
} }
write_terminator(write_stream, result_file_length, last_copy_pos); write_terminator(write_stream);
END_PROFILE(overall, "total patch building operation"); END_PROFILE(overall, "total patch building operation");
return (last_copy_pos != 0); if (express_cat.is_debug()) {
express_cat.debug()
<< "Patch file will generate " << _add_pos << "-byte file.\n";
}
// nassertr(_add_pos == result_file_length, false);
return (_last_copy_pos != 0);
} }
#endif // HAVE_OPENSSL #endif // HAVE_OPENSSL

45
panda/src/express/patchfile.h
View File

@ -94,30 +94,24 @@ private:
PN_uint32 calc_match_length(const char* buf1, const char* buf2, PN_uint32 max_length, PN_uint32 calc_match_length(const char* buf1, const char* buf2, PN_uint32 max_length,
PN_uint32 min_length); PN_uint32 min_length);
void emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer, void emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer);
PN_uint32 ADD_pos); void emit_COPY(ostream &write_stream, PN_uint32 length, PN_uint32 COPY_pos);
PN_uint32 emit_COPY(ostream &write_stream, PN_uint32 length, void emit_add_and_copy(ostream &write_stream,
PN_uint32 COPY_pos, PN_uint32 last_copy_pos, PN_uint32 add_length, const char *add_buffer,
PN_uint32 ADD_pos); PN_uint32 copy_length, PN_uint32 copy_pos);
PN_uint32 emit_add_and_copy(ostream &write_stream, void cache_add_and_copy(ostream &write_stream,
PN_uint32 add_length, const char *add_buffer, PN_uint32 add_length, const char *add_buffer,
PN_uint32 copy_length, PN_uint32 copy_pos, PN_uint32 last_copy_pos, PN_uint32 copy_length, PN_uint32 copy_pos);
PN_uint32 add_pos); void cache_flush(ostream &write_stream);
void write_header(ostream &write_stream, void write_header(ostream &write_stream,
char *buffer_orig, PN_uint32 source_file_length, istream &stream_orig, istream &stream_new);
char *buffer_new, PN_uint32 result_file_length); void write_terminator(ostream &write_stream);
void write_terminator(ostream &write_stream, PN_uint32 result_file_length,
PN_uint32 last_copy_pos);
PN_uint32 compute_patches(ostream &write_stream, PN_uint32 last_copy_pos, bool compute_patches(ostream &write_stream, PN_uint32 copy_offset,
PN_uint32 copy_offset, istream &stream_orig, istream &stream_new);
char *buffer_orig, PN_uint32 source_file_length, bool compute_mf_patches(ostream &write_stream,
char *buffer_new, PN_uint32 result_file_length); istream &stream_orig, istream &stream_new);
PN_uint32 compute_mf_patches(ostream &write_stream,
char *buffer_orig, PN_uint32 source_file_length,
char *buffer_new, PN_uint32 result_file_length);
static const PN_uint32 _HASH_BITS; static const PN_uint32 _HASH_BITS;
static const PN_uint32 _HASHTABLESIZE; static const PN_uint32 _HASHTABLESIZE;
@ -129,6 +123,15 @@ private:
bool _allow_multifile; bool _allow_multifile;
PN_uint32 _footprint_length; PN_uint32 _footprint_length;
PN_uint32 *_hash_table;
PN_uint32 _add_pos;
PN_uint32 _last_copy_pos;
string _cache_add_data;
PN_uint32 _cache_copy_start;
PN_uint32 _cache_copy_length;
protected: protected:
PT(Buffer) _buffer; // this is the work buffer for apply -- used to prevent virtual memory swapping PT(Buffer) _buffer; // this is the work buffer for apply -- used to prevent virtual memory swapping

3
panda/src/framework/windowFramework.cxx
View File

@ -190,7 +190,8 @@ open_window(const WindowProperties &props, GraphicsEngine *engine,
if (show_frame_rate_meter) { if (show_frame_rate_meter) {
_frame_rate_meter = new FrameRateMeter("frame_rate_meter"); _frame_rate_meter = new FrameRateMeter("frame_rate_meter");
_frame_rate_meter->setup_window(_window); // _frame_rate_meter->setup_window(_window);
_panda_framework->get_models().attach_new_node(_frame_rate_meter);
} }
} }