more fixes and optimizations to subfile patcher

This commit is contained in:
David Rose 2007-02-28 01:48:27 +00:00
parent 13c2658325
commit e51797e318
7 changed files with 451 additions and 218 deletions

View File

@ -32,8 +32,9 @@
bool create = false; // -c
bool append = false; // -r
bool update = false; // -u
bool tlist = false; // -t
bool tlist = false; // -t
bool extract = false; // -x
bool kill = false; // -k
bool verbose = false; // -v
bool compress_flag = false; // -z
int default_compression_level = 6;
@ -138,9 +139,13 @@ help() {
" Extract the contents of an existing Multifile. The Subfiles named on\n"
" the command line, or all Subfiles if nothing is named, are extracted\n"
" into the current directory or into whichever directory is specified\n"
" with -C.\n\n\n"
" with -C.\n\n"
" -k\n"
" Delete (kill) the named Subfiles from the Multifile. The Multifile\n"
" will be repacked after completion.\n\n"
"\n"
" You must always specify the following switch:\n\n"
" -f <multifile_name>\n"
@ -437,6 +442,41 @@ extract_files(int argc, char *argv[]) {
return true;
}
bool
kill_files(int argc, char *argv[]) {
if (!multifile_name.exists()) {
cerr << multifile_name << " not found.\n";
return false;
}
PT(Multifile) multifile = new Multifile;
if (!multifile->open_read_write(multifile_name)) {
cerr << "Unable to open " << multifile_name << " for read/write.\n";
return false;
}
int i = 0;
while (i < multifile->get_num_subfiles()) {
string subfile_name = multifile->get_subfile_name(i);
if (is_named(subfile_name, argc, argv)) {
Filename filename = subfile_name;
if (verbose) {
cout << filename << "\n";
}
multifile->remove_subfile(i);
} else {
++i;
}
}
if (!multifile->repack()) {
cerr << "Failed to write " << multifile_name << ".\n";
return false;
}
return true;
}
const char *
format_timestamp(bool record_timestamp, time_t timestamp) {
static const size_t buffer_size = 512;
@ -582,7 +622,7 @@ main(int argc, char *argv[]) {
extern char *optarg;
extern int optind;
static const char *optflags = "crutxvz123456789Z:T:f:OC:ep:F:h";
static const char *optflags = "crutxkvz123456789Z:T:f:OC:ep:F:h";
int flag = getopt(argc, argv, optflags);
Filename rel_path;
while (flag != EOF) {
@ -602,6 +642,9 @@ main(int argc, char *argv[]) {
case 'x':
extract = true;
break;
case 'k':
kill = true;
break;
case 'v':
verbose = true;
break;
@ -711,8 +754,8 @@ main(int argc, char *argv[]) {
argv += (optind - 1);
// We should have exactly one of these options.
if ((create?1:0) + (append?1:0) + (update?1:0) + (tlist?1:0) + (extract?1:0) != 1) {
cerr << "Exactly one of -c, -r, -u, -t, -x must be specified.\n";
if ((create?1:0) + (append?1:0) + (update?1:0) + (tlist?1:0) + (extract?1:0) + (kill?1:0) != 1) {
cerr << "Exactly one of -c, -r, -u, -t, -x, -k must be specified.\n";
usage();
return 1;
}
@ -734,6 +777,11 @@ main(int argc, char *argv[]) {
cerr << "Warning: -T ignored on extract.\n";
}
okflag = extract_files(argc, argv);
} else if (kill) {
if (got_record_timestamp_flag) {
cerr << "Warning: -T ignored on kill.\n";
}
okflag = kill_files(argc, argv);
} else { // list
if (got_record_timestamp_flag) {
cerr << "Warning: -T ignored on list.\n";

View File

@ -211,6 +211,25 @@ hash_file(const Filename &filename) {
return false;
}
bool result = hash_stream(*istr);
vfs->close_read_file(istr);
return result;
}
#endif // HAVE_OPENSSL
#ifdef HAVE_OPENSSL
////////////////////////////////////////////////////////////////////
// Function: HashVal::hash_stream
// Access: Published
// Description: Generates the hash value from the indicated file.
// Returns true on success, false if the file cannot be
// read. This method is only defined if we have the
// OpenSSL library (which provides md5 functionality)
// available.
////////////////////////////////////////////////////////////////////
bool HashVal::
hash_stream(istream &stream) {
unsigned char md[16];
MD5_CTX ctx;
@ -219,15 +238,21 @@ hash_file(const Filename &filename) {
static const int buffer_size = 1024;
char buffer[buffer_size];
istr->read(buffer, buffer_size);
size_t count = istr->gcount();
// Seek the stream to the beginning in case it wasn't there already.
stream.seekg(0, ios::beg);
stream.read(buffer, buffer_size);
size_t count = stream.gcount();
while (count != 0) {
MD5_Update(&ctx, buffer, count);
istr->read(buffer, buffer_size);
count = istr->gcount();
stream.read(buffer, buffer_size);
count = stream.gcount();
}
vfs->close_read_file(istr);
// Clear the fail bit so the caller can still read the stream (if it
// wants to).
stream.clear();
MD5_Final(md, &ctx);
// Store the individual bytes as big-endian ints, from historical
@ -239,7 +264,10 @@ hash_file(const Filename &filename) {
return true;
}
#endif // HAVE_OPENSSL
#ifdef HAVE_OPENSSL
////////////////////////////////////////////////////////////////////
// Function: HashVal::hash_buffer
// Access: Published

View File

@ -71,6 +71,7 @@ PUBLISHED:
#ifdef HAVE_OPENSSL
bool hash_file(const Filename &filename);
bool hash_stream(istream &stream);
INLINE void hash_ramfile(const Ramfile &ramfile);
INLINE void hash_string(const string &data);
void hash_buffer(const char *buffer, int length);

View File

@ -1161,6 +1161,7 @@ open_read(istream *multifile_stream) {
_timestamp = time(NULL);
_timestamp_dirty = true;
_read = multifile_stream;
_read->seekg(0, ios::beg);
return read_index();
}
@ -1180,6 +1181,7 @@ open_write(ostream *multifile_stream) {
_timestamp = time(NULL);
_timestamp_dirty = true;
_write = multifile_stream;
_write->seekp(0, ios::beg);
return true;
}
@ -1201,6 +1203,7 @@ open_read_write(iostream *multifile_stream) {
_timestamp_dirty = true;
_read = multifile_stream;
_write = multifile_stream;
_write->seekp(0, ios::beg);
// Check whether the read stream is empty.
_read->seekg(0, ios::end);
@ -1211,7 +1214,7 @@ open_read_write(iostream *multifile_stream) {
// The read stream is not empty, so we'd better have a valid
// Multifile.
_read->seekg(0);
_read->seekg(0, ios::beg);
return read_index();
}

View File

@ -26,6 +26,7 @@
#include "streamReader.h"
#include "streamWriter.h"
#include "multifile.h"
#include "hashVal.h"
#include <stdio.h> // for tempnam
@ -109,6 +110,7 @@ const PN_uint32 Patchfile::_HASH_MASK = (PN_uint32(1) << Patchfile::_HASH_BITS)
////////////////////////////////////////////////////////////////////
Patchfile::
Patchfile() {
_hash_table = NULL;
PT(Buffer) buffer = new Buffer(patchfile_buffer_size);
init(buffer);
}
@ -120,6 +122,7 @@ Patchfile() {
////////////////////////////////////////////////////////////////////
Patchfile::
Patchfile(PT(Buffer) buffer) {
_hash_table = NULL;
init(buffer);
}
@ -146,6 +149,10 @@ init(PT(Buffer) buffer) {
////////////////////////////////////////////////////////////////////
Patchfile::
~Patchfile() {
if (_hash_table != (PN_uint32 *)NULL) {
delete[] _hash_table;
}
if (true == _initiated)
cleanup();
}
@ -299,7 +306,7 @@ run() {
_total_bytes_processed += (int)ADD_length;
// if there are bytes to add, read them from patch file and write them to output
if (express_cat.is_spam()) {
if (express_cat.is_spam() && ADD_length != 0) {
express_cat.spam()
<< "ADD: " << ADD_length << " (to "
<< _write_stream.tellp() << ")" << endl;
@ -762,14 +769,12 @@ find_longest_match(PN_uint32 new_pos, PN_uint32 &copy_pos, PN_uint16 &copy_lengt
// Description:
////////////////////////////////////////////////////////////////////
void Patchfile::
emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer,
PN_uint32 ADD_pos) {
emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer) {
nassertv(length == (PN_uint16)length); //we only write a uint16
if (express_cat.is_spam()) {
express_cat.spam()
<< "ADD: " << length << " (to " << ADD_pos << ")" << endl;
<< "ADD: " << length << " (to " << _add_pos << ")" << endl;
}
// write ADD length
@ -780,6 +785,8 @@ emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer,
if (length > 0) {
patch_writer.append_data(buffer, (PN_uint16)length);
}
_add_pos += length;
}
////////////////////////////////////////////////////////////////////
@ -787,29 +794,28 @@ emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer,
// Access: Private
// Description:
////////////////////////////////////////////////////////////////////
PN_uint32 Patchfile::
emit_COPY(ostream &write_stream, PN_uint32 length, PN_uint32 COPY_pos,
PN_uint32 last_copy_pos, PN_uint32 ADD_pos) {
void Patchfile::
emit_COPY(ostream &write_stream, PN_uint32 length, PN_uint32 copy_pos) {
nassertv(length == (PN_uint16)length); //we only write a uint16
nassertr(length == (PN_uint16)length, last_copy_pos); //we only write a uint16
PN_int32 offset = (int)COPY_pos - (int)last_copy_pos;
PN_int32 offset = (int)copy_pos - (int)_last_copy_pos;
if (express_cat.is_spam()) {
express_cat.spam()
<< "COPY: " << length << " bytes from offset " << offset
<< " (from " << COPY_pos << " to " << ADD_pos << ")" << endl;
<< " (from " << copy_pos << " to " << _add_pos << ")" << endl;
}
// write COPY length
StreamWriter patch_writer(write_stream);
patch_writer.add_uint16((PN_uint16)length);
if((PN_uint16)length > 0) {
if ((PN_uint16)length != 0) {
// write COPY offset
patch_writer.add_int32(offset);
_last_copy_pos = copy_pos + length;
}
return COPY_pos + length;
_add_pos += length;
}
////////////////////////////////////////////////////////////////////
@ -819,42 +825,91 @@ emit_COPY(ostream &write_stream, PN_uint32 length, PN_uint32 COPY_pos,
// pair as needed to work around the 16-bit chunk size
// limit.
////////////////////////////////////////////////////////////////////
PN_uint32 Patchfile::
void Patchfile::
emit_add_and_copy(ostream &write_stream,
PN_uint32 add_length, const char *add_buffer,
PN_uint32 copy_length, PN_uint32 copy_pos, PN_uint32 last_copy_pos,
PN_uint32 add_pos) {
while (add_length > 65535) {
PN_uint32 copy_length, PN_uint32 copy_pos) {
if (add_length == 0 && copy_length == 0) {
// Don't accidentally emit a termination code.
return;
}
static const PN_uint16 max_write = 65535;
while (add_length > max_write) {
// Overflow. This chunk is too large to fit into a single
// ADD block, so we have to write it as multiple ADDs.
static const PN_uint16 max_write = 65535;
emit_ADD(write_stream, max_write, add_buffer, add_pos);
add_pos += max_write;
emit_ADD(write_stream, max_write, add_buffer);
add_buffer += max_write;
add_length -= max_write;
emit_COPY(write_stream, 0, last_copy_pos, last_copy_pos, add_pos);
emit_COPY(write_stream, 0, 0);
}
emit_ADD(write_stream, add_length, add_buffer, add_pos);
emit_ADD(write_stream, add_length, add_buffer);
while (copy_length > 65535) {
while (copy_length > max_write) {
// Overflow.
static const PN_uint16 max_write = 65535;
last_copy_pos =
emit_COPY(write_stream, max_write, copy_pos, last_copy_pos, add_pos);
emit_COPY(write_stream, max_write, copy_pos);
copy_pos += max_write;
add_pos += max_write;
copy_length -= max_write;
emit_ADD(write_stream, 0, NULL, add_pos);
emit_ADD(write_stream, 0, NULL);
}
last_copy_pos =
emit_COPY(write_stream, copy_length, copy_pos, last_copy_pos, add_pos);
return last_copy_pos;
emit_COPY(write_stream, copy_length, copy_pos);
}
////////////////////////////////////////////////////////////////////
// Function: Patchfile::cache_add_and_copy
// Access: Private
// Description: Potentially emits one or more add/copy pairs. The
// current state is saved, so as to minimize wasted
// emits from consecutive adds or copies.
////////////////////////////////////////////////////////////////////
void Patchfile::
cache_add_and_copy(ostream &write_stream,
PN_uint32 add_length, const char *add_buffer,
PN_uint32 copy_length, PN_uint32 copy_pos) {
if (add_length != 0) {
if (_cache_copy_length != 0) {
// Have to flush.
cache_flush(write_stream);
}
// Add the string to the current cache.
_cache_add_data += string(add_buffer, add_length);
}
if (copy_length != 0) {
if (_cache_copy_length == 0) {
// Start a new copy phase.
_cache_copy_start = copy_pos;
_cache_copy_length = copy_length;
} else if (_cache_copy_start + _cache_copy_length == copy_pos) {
// We can just tack on the copy to what we've already got.
_cache_copy_length += copy_length;
} else {
// It's a discontinuous copy. We have to flush.
cache_flush(write_stream);
_cache_copy_start = copy_pos;
_cache_copy_length = copy_length;
}
}
}
////////////////////////////////////////////////////////////////////
// Function: Patchfile::cache_flush
// Access: Private
// Description: Closes any copy or add phases that are still open
// after a previous call to cache_add_and_copy().
////////////////////////////////////////////////////////////////////
void Patchfile::
cache_flush(ostream &write_stream) {
emit_add_and_copy(write_stream,
_cache_add_data.size(), _cache_add_data.data(),
_cache_copy_length, _cache_copy_start);
_cache_add_data = string();
_cache_copy_length = 0;
}
////////////////////////////////////////////////////////////////////
@ -865,8 +920,7 @@ emit_add_and_copy(ostream &write_stream,
////////////////////////////////////////////////////////////////////
void Patchfile::
write_header(ostream &write_stream,
char *buffer_orig, PN_uint32 source_file_length,
char *buffer_new, PN_uint32 result_file_length) {
istream &stream_orig, istream &stream_new) {
// prepare to write the patch file header
START_PROFILE(writeHeader);
@ -875,19 +929,33 @@ write_header(ostream &write_stream,
StreamWriter patch_writer(write_stream);
patch_writer.add_uint32(_magic_number);
patch_writer.add_uint16(_current_version);
patch_writer.add_uint32(source_file_length);
{
// calc MD5 of original file
_MD5_ofSource.hash_buffer(buffer_orig, source_file_length);
// add it to the header
_MD5_ofSource.write_stream(patch_writer);
stream_orig.seekg(0, ios::end);
streampos source_file_length = stream_orig.tellg();
patch_writer.add_uint32((PN_uint32)source_file_length);
// calc MD5 of original file
_MD5_ofSource.hash_stream(stream_orig);
// add it to the header
_MD5_ofSource.write_stream(patch_writer);
if (express_cat.is_debug()) {
express_cat.debug()
<< "Orig: " << _MD5_ofSource << "\n";
}
patch_writer.add_uint32(result_file_length);
{
// calc MD5 of resultant patched file
_MD5_ofResult.hash_buffer(buffer_new, result_file_length);
// add it to the header
_MD5_ofResult.write_stream(patch_writer);
stream_new.seekg(0, ios::end);
streampos result_file_length = stream_new.tellg();
patch_writer.add_uint32((PN_uint32)result_file_length);
// calc MD5 of resultant patched file
_MD5_ofResult.hash_stream(stream_new);
// add it to the header
_MD5_ofResult.write_stream(patch_writer);
if (express_cat.is_debug()) {
express_cat.debug()
<< " New: " << _MD5_ofResult << "\n";
}
END_PROFILE(writeHeader, "writing patch file header");
@ -900,11 +968,11 @@ write_header(ostream &write_stream,
// Writes the patchfile terminator.
////////////////////////////////////////////////////////////////////
void Patchfile::
write_terminator(ostream &write_stream, PN_uint32 result_file_length,
PN_uint32 last_copy_pos) {
write_terminator(ostream &write_stream) {
cache_flush(write_stream);
// write terminator (null ADD, null COPY)
emit_ADD(write_stream, 0, NULL, result_file_length);
emit_COPY(write_stream, 0, last_copy_pos, last_copy_pos, result_file_length);
emit_ADD(write_stream, 0, NULL);
emit_COPY(write_stream, 0, 0);
}
////////////////////////////////////////////////////////////////////
@ -914,24 +982,48 @@ write_terminator(ostream &write_stream, PN_uint32 result_file_length,
// Computes the patches for the entire file (if it is
// not a multifile) or for a single subfile (if it is)
//
// Returns last_copy_pos, the last byte position from
// which we copied from the original file.
// Returns true if successful, false on error.
////////////////////////////////////////////////////////////////////
PN_uint32 Patchfile::
compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
PN_uint32 copy_offset,
char *buffer_orig, PN_uint32 source_file_length,
char *buffer_new, PN_uint32 result_file_length) {
bool Patchfile::
compute_patches(ostream &write_stream, PN_uint32 copy_offset,
istream &stream_orig, istream &stream_new) {
// read in original file
stream_orig.seekg(0, ios::end);
nassertr(stream_orig, false);
PN_uint32 source_file_length = stream_orig.tellg();
if (express_cat.is_debug()) {
express_cat.debug()
<< "Allocating " << source_file_length << " bytes to read orig\n";
}
char *buffer_orig = new char[source_file_length];
stream_orig.seekg(0, ios::beg);
stream_orig.read(buffer_orig, source_file_length);
// read in new file
stream_new.seekg(0, ios::end);
PN_uint32 result_file_length = stream_new.tellg();
nassertr(stream_new, false);
if (express_cat.is_debug()) {
express_cat.debug()
<< "Allocating " << result_file_length << " bytes to read new\n";
}
char *buffer_new = new char[result_file_length];
stream_new.seekg(0, ios::beg);
stream_new.read(buffer_new, result_file_length);
START_PROFILE(allocTables);
// allocate hash/link tables
if (express_cat.is_debug()) {
express_cat.debug()
<< "Allocating hashtable of size " << _HASHTABLESIZE << " * 4\n";
if (_hash_table == (PN_uint32 *)NULL) {
if (express_cat.is_debug()) {
express_cat.debug()
<< "Allocating hashtable of size " << _HASHTABLESIZE << " * 4\n";
}
_hash_table = new PN_uint32[_HASHTABLESIZE];
}
PN_uint32* hash_table = new PN_uint32[_HASHTABLESIZE];
if (express_cat.is_debug()) {
express_cat.debug()
<< "Allocating linktable of size " << source_file_length << " * 4\n";
@ -944,7 +1036,7 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
START_PROFILE(buildTables);
// build hash and link tables for original file
build_hash_link_tables(buffer_orig, source_file_length, hash_table, link_table);
build_hash_link_tables(buffer_orig, source_file_length, _hash_table, link_table);
END_PROFILE(buildTables, "building hash and link tables");
@ -952,7 +1044,7 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
START_PROFILE(buildPatchfile);
PN_uint32 new_pos = 0;
PN_uint32 ADD_pos = new_pos; // this is the position for the start of ADD operations
PN_uint32 start_pos = new_pos; // this is the position for the start of ADD operations
if(((PN_uint32) result_file_length) >= _footprint_length)
{
@ -962,7 +1054,7 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
PN_uint32 COPY_pos;
PN_uint16 COPY_length;
find_longest_match(new_pos, COPY_pos, COPY_length, hash_table, link_table,
find_longest_match(new_pos, COPY_pos, COPY_length, _hash_table, link_table,
buffer_orig, source_file_length, buffer_new, result_file_length);
// if no match or match not longer than footprint length, skip to next byte
@ -971,22 +1063,16 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
new_pos++;
} else {
// emit ADD for all skipped bytes
int num_skipped = (int)new_pos - (int)ADD_pos;
int num_skipped = (int)new_pos - (int)start_pos;
if (express_cat.is_spam()) {
express_cat.spam()
<< "build: num_skipped = " << num_skipped
<< endl;
}
last_copy_pos =
emit_add_and_copy(write_stream, num_skipped, &buffer_new[ADD_pos],
COPY_length, COPY_pos + copy_offset,
last_copy_pos, ADD_pos);
ADD_pos += num_skipped;
nassertr(ADD_pos == new_pos, last_copy_pos);
// skip past match in new_file
cache_add_and_copy(write_stream, num_skipped, &buffer_new[start_pos],
COPY_length, COPY_pos + copy_offset);
new_pos += (PN_uint32)COPY_length;
ADD_pos = new_pos;
start_pos = new_pos;
}
}
}
@ -994,28 +1080,28 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
if (express_cat.is_spam()) {
express_cat.spam()
<< "build: result_file_length = " << result_file_length
<< " ADD_pos = " << ADD_pos
<< " start_pos = " << start_pos
<< endl;
}
// are there still more bytes left in the new file?
if (ADD_pos != result_file_length) {
if (start_pos != result_file_length) {
// emit ADD for all remaining bytes
PN_uint32 remaining_bytes = result_file_length - ADD_pos;
last_copy_pos =
emit_add_and_copy(write_stream, remaining_bytes, &buffer_new[ADD_pos],
0, last_copy_pos, last_copy_pos, ADD_pos);
ADD_pos += remaining_bytes;
nassertr(ADD_pos == result_file_length, last_copy_pos);
PN_uint32 remaining_bytes = result_file_length - start_pos;
cache_add_and_copy(write_stream, remaining_bytes, &buffer_new[start_pos],
0, 0);
start_pos += remaining_bytes;
}
END_PROFILE(buildPatchfile, "building patch file");
delete[] hash_table;
delete[] link_table;
return last_copy_pos;
delete[] buffer_orig;
delete[] buffer_new;
return true;
}
////////////////////////////////////////////////////////////////////
@ -1028,36 +1114,33 @@ compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
// much faster for large Multifiles that contain many
// small subfiles.
////////////////////////////////////////////////////////////////////
PN_uint32 Patchfile::
bool Patchfile::
compute_mf_patches(ostream &write_stream,
char *buffer_orig, PN_uint32 source_file_length,
char *buffer_new, PN_uint32 result_file_length) {
string string_orig(buffer_orig, source_file_length);
string string_new(buffer_new, result_file_length);
istringstream strm_orig(string_orig);
istringstream strm_new(string_new);
istream &stream_orig, istream &stream_new) {
Multifile mf_orig, mf_new;
if (!mf_orig.open_read(&strm_orig) ||
!mf_new.open_read(&strm_new)) {
if (!mf_orig.open_read(&stream_orig) ||
!mf_new.open_read(&stream_new)) {
express_cat.error()
<< "Input multifiles appear to be corrupt.\n";
return 0;
return false;
}
if (mf_new.needs_repack()) {
express_cat.error()
<< "Input multifiles need to be repacked.\n";
return 0;
return false;
}
// First, compute the patch for the header / index.
PN_uint32 last_copy_pos =
compute_patches(write_stream, 0, 0,
buffer_orig, (PN_uint32)mf_orig.get_index_end(),
buffer_new, (PN_uint32)mf_new.get_index_end());
PN_uint32 add_pos = (PN_uint32)mf_new.get_index_end();
{
ISubStream index_orig(&stream_orig, 0, mf_orig.get_index_end());
ISubStream index_new(&stream_new, 0, mf_new.get_index_end());
if (!compute_patches(write_stream, 0, index_orig, index_new)) {
return false;
}
nassertr(_add_pos + _cache_add_data.size() + _cache_copy_length == mf_new.get_index_end(), false);
}
// Now walk through each subfile in the new multifile. If a
// particular subfile exists in both source files, we compute the
@ -1066,54 +1149,128 @@ compute_mf_patches(ostream &write_stream,
// never even notice this case).
int new_num_subfiles = mf_new.get_num_subfiles();
for (int ni = 0; ni < new_num_subfiles; ++ni) {
nassertr(add_pos == mf_new.get_subfile_internal_start(ni), last_copy_pos);
nassertr(_add_pos + _cache_add_data.size() + _cache_copy_length == mf_new.get_subfile_internal_start(ni), false);
string name = mf_new.get_subfile_name(ni);
int oi = mf_orig.find_subfile(name);
/*
if (oi < 0) {
// This subfile exists in the new file, but not in the original
// file. Trivially add it.
string new_ext = Filename(name).get_extension();
if (new_ext != "jpg") {
// This is a newly-added subfile. Look for another subfile with
// the same extension, so we can generate a patch against that
// other subfile--the idea is that there are likely to be
// similar byte sequences in files with the same extension, so
// "patching" a new subfile against a different subfile may come
// out smaller than baldly adding the new subfile.
size_t new_size = mf_new.get_subfile_internal_length(ni);
int orig_num_subfiles = mf_orig.get_num_subfiles();
size_t best_size = 0;
for (int i = 0; i < orig_num_subfiles; ++i) {
string orig_ext = Filename(mf_orig.get_subfile_name(i)).get_extension();
if (orig_ext == new_ext) {
size_t orig_size = mf_orig.get_subfile_internal_length(i);
// Find the smallest candidate that is no smaller than our
// target file. If all the candidates are smaller than our
// target file, choose the largest of them.
if ((best_size < new_size && orig_size > best_size) ||
(best_size >= new_size && orig_size >= new_size && orig_size < best_size)) {
best_size = orig_size;
oi = i;
}
}
}
}
}
*/
if (oi < 0) {
// This is a newly-added subfile, and we didn't find another
// subfile with a matching extension. Add it the hard way.
express_cat.info()
<< "Adding subfile " << mf_new.get_subfile_name(ni) << "\n";
PN_uint32 new_start = (PN_uint32)mf_new.get_subfile_internal_start(ni);
PN_uint32 new_size = (PN_uint32)mf_new.get_subfile_internal_length(ni);
last_copy_pos =
emit_add_and_copy(write_stream, new_size, &buffer_new[new_start],
0, last_copy_pos, last_copy_pos, add_pos);
add_pos += new_size;
streampos new_start = mf_new.get_subfile_internal_start(ni);
size_t new_size = mf_new.get_subfile_internal_length(ni);
char *buffer_new = new char[new_size];
stream_new.seekg(new_start, ios::beg);
stream_new.read(buffer_new, new_size);
cache_add_and_copy(write_stream, new_size, buffer_new, 0, 0);
delete[] buffer_new;
} else {
// This subfile exists in both the original and the new files.
// Patch it.
PN_uint32 orig_start = (PN_uint32)mf_orig.get_subfile_internal_start(oi);
PN_uint32 orig_size = (PN_uint32)mf_orig.get_subfile_internal_length(oi);
PN_uint32 new_start = (PN_uint32)mf_new.get_subfile_internal_start(ni);
PN_uint32 new_size = (PN_uint32)mf_new.get_subfile_internal_length(ni);
if (orig_size == new_size &&
memcmp(&buffer_orig[orig_start], &buffer_new[new_start], new_size) == 0) {
// Actually, the subfile is unchanged; just emit it.
streampos orig_start = mf_orig.get_subfile_internal_start(oi);
size_t orig_size = mf_orig.get_subfile_internal_length(oi);
ISubStream subfile_orig(&stream_orig, orig_start, orig_start + (streampos)orig_size);
streampos new_start = mf_new.get_subfile_internal_start(ni);
size_t new_size = mf_new.get_subfile_internal_length(ni);
ISubStream subfile_new(&stream_new, new_start, new_start + (streampos)new_size);
bool is_unchanged = false;
if (orig_size == new_size) {
HashVal hash_orig, hash_new;
hash_orig.hash_stream(subfile_orig);
hash_new.hash_stream(subfile_new);
if (hash_orig == hash_new) {
// Actually, the subfile is unchanged; just emit it.
is_unchanged = true;
}
}
if (is_unchanged) {
if (express_cat.is_debug()) {
express_cat.debug()
<< "Keeping subfile " << mf_new.get_subfile_name(ni) << "\n";
<< "Keeping subfile " << mf_new.get_subfile_name(ni);
if (mf_orig.get_subfile_name(oi) != mf_new.get_subfile_name(ni)) {
express_cat.debug(false)
<< " (identical to " << mf_orig.get_subfile_name(oi) << ")";
}
express_cat.debug(false) << "\n";
}
last_copy_pos =
emit_add_and_copy(write_stream, 0, NULL,
new_size, orig_start, last_copy_pos,
add_pos);
cache_add_and_copy(write_stream, 0, NULL,
orig_size, orig_start);
} else {
express_cat.info()
<< "Patching subfile " << mf_new.get_subfile_name(ni) << "\n";
last_copy_pos =
compute_patches(write_stream, last_copy_pos, orig_start,
&buffer_orig[orig_start], orig_size,
&buffer_new[new_start], new_size);
<< "Patching subfile " << mf_new.get_subfile_name(ni);
if (mf_orig.get_subfile_name(oi) != mf_new.get_subfile_name(ni)) {
express_cat.info(false)
<< " (against " << mf_orig.get_subfile_name(oi) << ")";
}
express_cat.info(false) << "\n";
if (!compute_patches(write_stream, orig_start,
subfile_orig, subfile_new)) {
return false;
}
/*
// Simply copy the new file; don't attempt to patch.
if (express_cat.is_debug()) {
express_cat.debug()
<< "Copying subfile " << mf_new.get_subfile_name(ni) << "\n";
}
streampos new_start = mf_new.get_subfile_internal_start(ni);
size_t new_size = mf_new.get_subfile_internal_length(ni);
char *buffer_new = new char[new_size];
stream_new.seekg(new_start, ios::beg);
stream_new.read(buffer_new, new_size);
cache_add_and_copy(write_stream, new_size, buffer_new,
0, 0);
delete[] buffer_new;
*/
}
add_pos += new_size;
}
}
nassertr(add_pos == result_file_length, last_copy_pos);
return last_copy_pos;
return true;
}
////////////////////////////////////////////////////////////////////
@ -1134,8 +1291,6 @@ build(Filename file_orig, Filename file_new, Filename patch_name) {
START_PROFILE(overall);
START_PROFILE(readFiles);
// Open the original file for read
ifstream stream_orig;
file_orig.set_binary();
@ -1162,73 +1317,67 @@ build(Filename file_orig, Filename file_new, Filename patch_name) {
return false;
}
// read in original file
stream_orig.seekg(0, ios::end);
PN_uint32 source_file_length = stream_orig.tellg();
if (express_cat.is_debug()) {
express_cat.debug()
<< "Allocating " << source_file_length << " bytes to read "
<< file_orig << "\n";
}
_last_copy_pos = 0;
_add_pos = 0;
_cache_add_data = string();
_cache_copy_start = 0;
_cache_copy_length = 0;
char *buffer_orig = new char[source_file_length];
stream_orig.seekg(0, ios::beg);
stream_orig.read(buffer_orig, source_file_length);
// read in new file
stream_new.seekg(0, ios::end);
PN_uint32 result_file_length = stream_new.tellg();
if (express_cat.is_debug()) {
express_cat.debug()
<< "Allocating " << result_file_length << " bytes to write "
<< file_new << "\n";
}
char *buffer_new = new char[result_file_length];
stream_new.seekg(0, ios::beg);
stream_new.read(buffer_new, result_file_length);
// close the original and new files (we have em in memory)
stream_orig.close();
stream_new.close();
END_PROFILE(readFiles, "reading files");
write_header(write_stream, buffer_orig, source_file_length,
buffer_new, result_file_length);
PN_uint32 last_copy_pos;
write_header(write_stream, stream_orig, stream_new);
// Check whether our input files are Panda multifiles.
bool is_multifile = false;
if (_allow_multifile) {
if (file_orig.get_extension() == "mf" || file_new.get_extension() == "mf") {
// Read the first n bytes of both files for the Multifile magic
// number.
string magic_number = Multifile::get_magic_number();
if (source_file_length > magic_number.size() &&
result_file_length > magic_number.size() &&
memcmp(buffer_orig, magic_number.data(), magic_number.size()) == 0 &&
memcmp(buffer_new, magic_number.data(), magic_number.size()) == 0) {
is_multifile = true;
char *buffer = new char[magic_number.size()];
stream_orig.seekg(0, ios::beg);
stream_orig.read(buffer, magic_number.size());
if (stream_orig.gcount() == magic_number.size() &&
memcmp(buffer, magic_number.data(), magic_number.size()) == 0) {
stream_new.seekg(0, ios::beg);
stream_new.read(buffer, magic_number.size());
if (stream_new.gcount() == magic_number.size() &&
memcmp(buffer, magic_number.data(), magic_number.size()) == 0) {
is_multifile = true;
}
}
delete[] buffer;
}
}
if (is_multifile) {
last_copy_pos =
compute_mf_patches(write_stream, buffer_orig, source_file_length,
buffer_new, result_file_length);
if (express_cat.is_debug()) {
express_cat.debug()
<< "Input files appear to be Panda Multifiles.\n";
}
if (!compute_mf_patches(write_stream, stream_orig, stream_new)) {
return false;
}
} else {
last_copy_pos =
compute_patches(write_stream, 0, 0,
buffer_orig, source_file_length,
buffer_new, result_file_length);
if (express_cat.is_debug()) {
express_cat.debug()
<< "Input files are NOT Panda Multifiles.\n";
}
if (!compute_patches(write_stream, 0, stream_orig, stream_new)) {
return false;
}
}
write_terminator(write_stream, result_file_length, last_copy_pos);
write_terminator(write_stream);
END_PROFILE(overall, "total patch building operation");
return (last_copy_pos != 0);
if (express_cat.is_debug()) {
express_cat.debug()
<< "Patch file will generate " << _add_pos << "-byte file.\n";
}
// nassertr(_add_pos == result_file_length, false);
return (_last_copy_pos != 0);
}
#endif // HAVE_OPENSSL

View File

@ -94,30 +94,24 @@ private:
PN_uint32 calc_match_length(const char* buf1, const char* buf2, PN_uint32 max_length,
PN_uint32 min_length);
void emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer,
PN_uint32 ADD_pos);
PN_uint32 emit_COPY(ostream &write_stream, PN_uint32 length,
PN_uint32 COPY_pos, PN_uint32 last_copy_pos,
PN_uint32 ADD_pos);
PN_uint32 emit_add_and_copy(ostream &write_stream,
PN_uint32 add_length, const char *add_buffer,
PN_uint32 copy_length, PN_uint32 copy_pos, PN_uint32 last_copy_pos,
PN_uint32 add_pos);
void emit_ADD(ostream &write_stream, PN_uint32 length, const char* buffer);
void emit_COPY(ostream &write_stream, PN_uint32 length, PN_uint32 COPY_pos);
void emit_add_and_copy(ostream &write_stream,
PN_uint32 add_length, const char *add_buffer,
PN_uint32 copy_length, PN_uint32 copy_pos);
void cache_add_and_copy(ostream &write_stream,
PN_uint32 add_length, const char *add_buffer,
PN_uint32 copy_length, PN_uint32 copy_pos);
void cache_flush(ostream &write_stream);
void write_header(ostream &write_stream,
char *buffer_orig, PN_uint32 source_file_length,
char *buffer_new, PN_uint32 result_file_length);
void write_terminator(ostream &write_stream, PN_uint32 result_file_length,
PN_uint32 last_copy_pos);
istream &stream_orig, istream &stream_new);
void write_terminator(ostream &write_stream);
PN_uint32 compute_patches(ostream &write_stream, PN_uint32 last_copy_pos,
PN_uint32 copy_offset,
char *buffer_orig, PN_uint32 source_file_length,
char *buffer_new, PN_uint32 result_file_length);
PN_uint32 compute_mf_patches(ostream &write_stream,
char *buffer_orig, PN_uint32 source_file_length,
char *buffer_new, PN_uint32 result_file_length);
bool compute_patches(ostream &write_stream, PN_uint32 copy_offset,
istream &stream_orig, istream &stream_new);
bool compute_mf_patches(ostream &write_stream,
istream &stream_orig, istream &stream_new);
static const PN_uint32 _HASH_BITS;
static const PN_uint32 _HASHTABLESIZE;
@ -129,6 +123,15 @@ private:
bool _allow_multifile;
PN_uint32 _footprint_length;
PN_uint32 *_hash_table;
PN_uint32 _add_pos;
PN_uint32 _last_copy_pos;
string _cache_add_data;
PN_uint32 _cache_copy_start;
PN_uint32 _cache_copy_length;
protected:
PT(Buffer) _buffer; // this is the work buffer for apply -- used to prevent virtual memory swapping

View File

@ -190,7 +190,8 @@ open_window(const WindowProperties &props, GraphicsEngine *engine,
if (show_frame_rate_meter) {
_frame_rate_meter = new FrameRateMeter("frame_rate_meter");
_frame_rate_meter->setup_window(_window);
// _frame_rate_meter->setup_window(_window);
_panda_framework->get_models().attach_new_node(_frame_rate_meter);
}
}