Fix handling of large (>2GB) files on Windows. Egad, off_t is 32 bits on Windows, even in 64-bit Windows. Ridiculous.

This commit is contained in:
David Rose 2013-08-01 00:10:20 +00:00
parent 574199c94d
commit dba661253a
32 changed files with 150 additions and 79 deletions

View File

@ -1662,14 +1662,17 @@ get_access_timestamp() const {
// Description: Returns the size of the file in bytes, or 0 if there
// is an error.
////////////////////////////////////////////////////////////////////
off_t Filename::
streamsize Filename::
get_file_size() const {
#ifdef WIN32_VC
wstring os_specific = get_filename_index(0).to_os_specific_w();
struct _stat this_buf;
struct _stat64 this_buf;
if (_wstat(os_specific.c_str(), &this_buf) == 0) {
// _wstat() only returns the lower 32 bits of the file size (!). We
// have to call _wstati64() if we actually want the full 64-bit file
// size.
if (_wstati64(os_specific.c_str(), &this_buf) == 0) {
return this_buf.st_size;
}
#else // WIN32_VC

View File

@ -184,7 +184,7 @@ PUBLISHED:
bool other_missing_is_old = true) const;
time_t get_timestamp() const;
time_t get_access_timestamp() const;
off_t get_file_size() const;
streamsize get_file_size() const;
bool resolve_filename(const DSearchPath &searchpath,
const string &default_extension = string());

View File

@ -319,17 +319,17 @@ seekoff(streamoff off, ios_seekdir dir, ios_openmode which) {
gbump(n);
_gpos -= n;
assert(_gpos >= 0);
size_t cur_pos = _gpos;
size_t new_pos = cur_pos;
streampos cur_pos = _gpos;
streampos new_pos = cur_pos;
// Now adjust the data pointer appropriately.
switch (dir) {
case ios::beg:
new_pos = (size_t)off;
new_pos = (streampos)off;
break;
case ios::cur:
new_pos = (size_t)((int)cur_pos + off);
new_pos = (streampos)(cur_pos + off);
break;
case ios::end:
@ -375,7 +375,7 @@ seekoff(streamoff off, ios_seekdir dir, ios_openmode which) {
break;
case ios::cur:
new_pos = (streampos)((int)cur_pos + off);
new_pos = (streampos)(cur_pos + off);
break;
case ios::end:
@ -440,7 +440,7 @@ overflow(int ch) {
size_t n = pptr() - pbase();
if (n != 0) {
size_t wrote = write_chars(pbase(), n);
pbump(-(int)wrote);
pbump(-(ssize_t)wrote);
if (wrote != n) {
okflag = false;
}
@ -474,7 +474,7 @@ sync() {
size_t n = pptr() - pbase();
size_t wrote = write_chars(pbase(), n);
pbump(-(int)wrote);
pbump(-(ssize_t)wrote);
if (n != wrote) {
return EOF;
@ -496,7 +496,7 @@ underflow() {
// Mark the buffer filled (with buffer_size bytes).
size_t buffer_size = egptr() - eback();
gbump(-(int)buffer_size);
gbump(-(ssize_t)buffer_size);
size_t num_bytes = buffer_size;
size_t read_count = read_chars(gptr(), buffer_size);
@ -707,7 +707,7 @@ read_chars_raw(char *start, size_t length) {
return 0;
}
ssize_t result = ::read(_fd, start, length);
int result = ::read(_fd, start, length);
while (result < 0) {
if (errno == EAGAIN) {
thread_yield();
@ -796,7 +796,7 @@ write_chars_raw(const char *start, size_t length) {
size_t remaining = length;
while (remaining > 0) {
ssize_t result = ::write(_fd, start, remaining);
int result = ::write(_fd, start, remaining);
if (result < 0) {
if (errno == EAGAIN) {
thread_yield();

View File

@ -1005,7 +1005,7 @@ load(const Filename &file_name) {
bool is_midi_file = (downcase(extension) == "mid");
if ((miles_audio_preload_threshold == -1 || file->get_file_size() < (off_t)miles_audio_preload_threshold) ||
if ((miles_audio_preload_threshold == -1 || file->get_file_size() < (streamsize)miles_audio_preload_threshold) ||
is_midi_file) {
// If the file is sufficiently small, we'll preload it into
// memory. MIDI files cannot be streamed, so we always preload

View File

@ -267,7 +267,7 @@ will_close_connection() const {
// their minds midstream about how much data they're
// sending you.
////////////////////////////////////////////////////////////////////
off_t HTTPChannel::
streamsize HTTPChannel::
get_file_size() const {
if (_got_file_size) {
return _file_size;

View File

@ -151,7 +151,7 @@ PUBLISHED:
INLINE double get_max_updates_per_second() const;
INLINE void set_expected_file_size(size_t file_size);
off_t get_file_size() const;
streamsize get_file_size() const;
INLINE bool is_file_size_known() const;
INLINE size_t get_first_byte_requested() const;

View File

@ -240,7 +240,7 @@ was_read_successful() const {
// implementations may require this stream to determine
// the size.
////////////////////////////////////////////////////////////////////
off_t VirtualFileHTTP::
streamsize VirtualFileHTTP::
get_file_size(istream *stream) const {
return _channel->get_file_size();
}
@ -251,7 +251,7 @@ get_file_size(istream *stream) const {
// Description: Returns the current size on disk (or wherever it is)
// of the file before it has been opened.
////////////////////////////////////////////////////////////////////
off_t VirtualFileHTTP::
streamsize VirtualFileHTTP::
get_file_size() const {
return _channel->get_file_size();
}

View File

@ -49,8 +49,8 @@ public:
virtual istream *open_read_file(bool auto_unwrap) const;
virtual bool was_read_successful() const;
virtual off_t get_file_size(istream *stream) const;
virtual off_t get_file_size() const;
virtual streamsize get_file_size(istream *stream) const;
virtual streamsize get_file_size() const;
virtual time_t get_timestamp() const;
private:

View File

@ -212,7 +212,7 @@ open_read_file(const Filename &) const {
// implementations may require this stream to determine
// the size.
////////////////////////////////////////////////////////////////////
off_t VirtualFileMountHTTP::
streamsize VirtualFileMountHTTP::
get_file_size(const Filename &, istream *) const {
return 0;
}
@ -223,7 +223,7 @@ get_file_size(const Filename &, istream *) const {
// Description: Returns the current size on disk (or wherever it is)
// of the file before it has been opened.
////////////////////////////////////////////////////////////////////
off_t VirtualFileMountHTTP::
streamsize VirtualFileMountHTTP::
get_file_size(const Filename &) const {
return 0;
}

View File

@ -52,8 +52,8 @@ public:
virtual bool is_regular_file(const Filename &file) const;
virtual istream *open_read_file(const Filename &file) const;
virtual off_t get_file_size(const Filename &file, istream *stream) const;
virtual off_t get_file_size(const Filename &file) const;
virtual streamsize get_file_size(const Filename &file, istream *stream) const;
virtual streamsize get_file_size(const Filename &file) const;
virtual time_t get_timestamp(const Filename &file) const;
virtual bool scan_directory(vector_string &contents,

View File

@ -347,7 +347,7 @@ close_read_write_file(iostream *stream) {
// implementations may require this stream to determine
// the size.
////////////////////////////////////////////////////////////////////
off_t VirtualFile::
streamsize VirtualFile::
get_file_size(istream *stream) const {
return get_file_size();
}
@ -358,7 +358,7 @@ get_file_size(istream *stream) const {
// Description: Returns the current size on disk (or wherever it is)
// of the file before it has been opened.
////////////////////////////////////////////////////////////////////
off_t VirtualFile::
streamsize VirtualFile::
get_file_size() const {
return 0;
}

View File

@ -72,8 +72,8 @@ PUBLISHED:
BLOCKING virtual iostream *open_read_append_file();
BLOCKING virtual void close_read_write_file(iostream *stream);
BLOCKING virtual off_t get_file_size(istream *stream) const;
BLOCKING virtual off_t get_file_size() const;
BLOCKING virtual streamsize get_file_size(istream *stream) const;
BLOCKING virtual streamsize get_file_size() const;
BLOCKING virtual time_t get_timestamp() const;
virtual bool get_system_info(SubfileInfo &info);

View File

@ -161,7 +161,7 @@ read_file(const Filename &file, bool do_uncompress,
return false;
}
off_t file_size = get_file_size(file, in);
streamsize file_size = get_file_size(file, in);
if (file_size > 0) {
result.reserve((size_t)file_size);
}

View File

@ -73,8 +73,8 @@ public:
virtual iostream *open_read_append_file(const Filename &file);
virtual void close_read_write_file(iostream *stream);
virtual off_t get_file_size(const Filename &file, istream *stream) const=0;
virtual off_t get_file_size(const Filename &file) const=0;
virtual streamsize get_file_size(const Filename &file, istream *stream) const=0;
virtual streamsize get_file_size(const Filename &file) const=0;
virtual time_t get_timestamp(const Filename &file) const=0;
virtual bool get_system_info(const Filename &file, SubfileInfo &info);

View File

@ -182,7 +182,7 @@ open_read_file(const Filename &file) const {
// implementations may require this stream to determine
// the size.
////////////////////////////////////////////////////////////////////
off_t VirtualFileMountAndroidAsset::
streamsize VirtualFileMountAndroidAsset::
get_file_size(const Filename &file, istream *in) const {
// If it's already open, get the AAsset pointer from the streambuf.
const AssetStreamBuf *buf = (const AssetStreamBuf *) in->rdbuf();
@ -196,7 +196,7 @@ get_file_size(const Filename &file, istream *in) const {
// Description: Returns the current size on disk (or wherever it is)
// of the file before it has been opened.
////////////////////////////////////////////////////////////////////
off_t VirtualFileMountAndroidAsset::
streamsize VirtualFileMountAndroidAsset::
get_file_size(const Filename &file) const {
AAsset* asset = AAssetManager_open(_asset_mgr, file.c_str(), AASSET_MODE_UNKNOWN);
off_t length = AAsset_getLength(asset);

View File

@ -46,8 +46,8 @@ public:
pvector<unsigned char> &result) const;
virtual istream *open_read_file(const Filename &file) const;
virtual off_t get_file_size(const Filename &file, istream *stream) const;
virtual off_t get_file_size(const Filename &file) const;
virtual streamsize get_file_size(const Filename &file, istream *stream) const;
virtual streamsize get_file_size(const Filename &file) const;
virtual time_t get_timestamp(const Filename &file) const;
virtual bool get_system_info(const Filename &file, SubfileInfo &info);

View File

@ -124,7 +124,7 @@ open_read_file(const Filename &file) const {
// implementations may require this stream to determine
// the size.
////////////////////////////////////////////////////////////////////
off_t VirtualFileMountMultifile::
streamsize VirtualFileMountMultifile::
get_file_size(const Filename &file, istream *) const {
int subfile_index = _multifile->find_subfile(file);
if (subfile_index < 0) {
@ -139,7 +139,7 @@ get_file_size(const Filename &file, istream *) const {
// Description: Returns the current size on disk (or wherever it is)
// of the file before it has been opened.
////////////////////////////////////////////////////////////////////
off_t VirtualFileMountMultifile::
streamsize VirtualFileMountMultifile::
get_file_size(const Filename &file) const {
int subfile_index = _multifile->find_subfile(file);
if (subfile_index < 0) {

View File

@ -42,8 +42,8 @@ public:
pvector<unsigned char> &result) const;
virtual istream *open_read_file(const Filename &file) const;
virtual off_t get_file_size(const Filename &file, istream *stream) const;
virtual off_t get_file_size(const Filename &file) const;
virtual streamsize get_file_size(const Filename &file, istream *stream) const;
virtual streamsize get_file_size(const Filename &file) const;
virtual time_t get_timestamp(const Filename &file) const;
virtual bool get_system_info(const Filename &file, SubfileInfo &info);

View File

@ -344,7 +344,7 @@ open_read_append_file(const Filename &file) {
// implementations may require this stream to determine
// the size.
////////////////////////////////////////////////////////////////////
off_t VirtualFileMountRamdisk::
streamsize VirtualFileMountRamdisk::
get_file_size(const Filename &file, istream *stream) const {
((VirtualFileMountRamdisk *)this)->_lock.acquire();
PT(FileBase) f = _root.do_find_file(file);
@ -363,7 +363,7 @@ get_file_size(const Filename &file, istream *stream) const {
// Description: Returns the current size on disk (or wherever it is)
// of the file before it has been opened.
////////////////////////////////////////////////////////////////////
off_t VirtualFileMountRamdisk::
streamsize VirtualFileMountRamdisk::
get_file_size(const Filename &file) const {
((VirtualFileMountRamdisk *)this)->_lock.acquire();
PT(FileBase) f = _root.do_find_file(file);

View File

@ -52,8 +52,8 @@ public:
virtual iostream *open_read_write_file(const Filename &file, bool truncate);
virtual iostream *open_read_append_file(const Filename &file);
virtual off_t get_file_size(const Filename &file, istream *stream) const;
virtual off_t get_file_size(const Filename &file) const;
virtual streamsize get_file_size(const Filename &file, istream *stream) const;
virtual streamsize get_file_size(const Filename &file) const;
virtual time_t get_timestamp(const Filename &file) const;
virtual bool scan_directory(vector_string &contents,

View File

@ -335,7 +335,7 @@ open_read_append_file(const Filename &file) {
// implementations may require this stream to determine
// the size.
////////////////////////////////////////////////////////////////////
off_t VirtualFileMountSystem::
streamsize VirtualFileMountSystem::
get_file_size(const Filename &file, istream *stream) const {
// First, save the original stream position.
streampos orig = stream->tellg();
@ -364,7 +364,7 @@ get_file_size(const Filename &file, istream *stream) const {
// Description: Returns the current size on disk (or wherever it is)
// of the file before it has been opened.
////////////////////////////////////////////////////////////////////
off_t VirtualFileMountSystem::
streamsize VirtualFileMountSystem::
get_file_size(const Filename &file) const {
Filename pathname(_physical_filename, file);
return pathname.get_file_size();

View File

@ -47,8 +47,8 @@ public:
virtual iostream *open_read_write_file(const Filename &file, bool truncate);
virtual iostream *open_read_append_file(const Filename &file);
virtual off_t get_file_size(const Filename &file, istream *stream) const;
virtual off_t get_file_size(const Filename &file) const;
virtual streamsize get_file_size(const Filename &file, istream *stream) const;
virtual streamsize get_file_size(const Filename &file) const;
virtual time_t get_timestamp(const Filename &file) const;
virtual bool get_system_info(const Filename &file, SubfileInfo &info);

View File

@ -354,7 +354,7 @@ close_read_write_file(iostream *stream) {
// implementations may require this stream to determine
// the size.
////////////////////////////////////////////////////////////////////
off_t VirtualFileSimple::
streamsize VirtualFileSimple::
get_file_size(istream *stream) const {
return _mount->get_file_size(_local_filename, stream);
}
@ -365,7 +365,7 @@ get_file_size(istream *stream) const {
// Description: Returns the current size on disk (or wherever it is)
// of the file before it has been opened.
////////////////////////////////////////////////////////////////////
off_t VirtualFileSimple::
streamsize VirtualFileSimple::
get_file_size() const {
return _mount->get_file_size(_local_filename);
}

View File

@ -57,8 +57,8 @@ PUBLISHED:
virtual iostream *open_read_append_file();
virtual void close_read_write_file(iostream *stream);
virtual off_t get_file_size(istream *stream) const;
virtual off_t get_file_size() const;
virtual streamsize get_file_size(istream *stream) const;
virtual streamsize get_file_size() const;
virtual time_t get_timestamp() const;
virtual bool get_system_info(SubfileInfo &info);

View File

@ -105,7 +105,9 @@ extrude(const Lens *lens) {
PfmFile result;
result.clear(_pfm.get_x_size(), _pfm.get_y_size(), 3);
result.set_zero_special(true);
if (_pfm.has_no_data_value()) {
result.set_zero_special(true);
}
if (lens->is_linear()) {
// If the lens is linear (Perspective or Orthographic), we can

View File

@ -293,25 +293,26 @@ pandavfs_open(URLContext *h, const char *filename, int flags) {
////////////////////////////////////////////////////////////////////
int FfmpegVirtualFile::
pandavfs_read(URLContext *h, unsigned char *buf, int size) {
streampos ssize = (streampos)size;
FfmpegVirtualFile *self = (FfmpegVirtualFile *)(h->priv_data);
istream *in = self->_in;
// Since we may be simulating a subset of the opened stream, don't
// allow it to read past the "end".
streampos remaining = self->_start + (streampos)self->_size - in->tellg();
if (remaining < size) {
if (remaining < ssize) {
if (remaining <= 0) {
return 0;
}
size = (int)remaining;
ssize = remaining;
}
in->read((char *)buf, size);
int gc = in->gcount();
in->read((char *)buf, ssize);
streamsize gc = in->gcount();
in->clear();
return gc;
return (int)gc;
}
////////////////////////////////////////////////////////////////////

View File

@ -1196,8 +1196,8 @@ lighten_sub_image(const PNMImage &copy, int xto, int yto,
//
// For each pixel (x, y):
//
// s = select_image.get_channel(x, y). Set this image's
// (x, y) to:
// s = select_image.get_channel(x, y, channel). Set this
// image's (x, y) to:
//
// lt.get_xel(x, y) if s < threshold, or
//
@ -1697,6 +1697,33 @@ expand_border(int left, int right, int bottom, int top,
take_from(new_image);
}
////////////////////////////////////////////////////////////////////
// Function: PNMImage::unfiltered_stretch_from
// Access: Published
// Description: Resizes from the indicated image into this one by
// performing a nearest-point sample.
////////////////////////////////////////////////////////////////////
void PNMImage::
unfiltered_stretch_from(const PNMImage &copy) {
for (int yt = 0; yt < get_y_size(); yt++) {
int ys = yt * copy.get_y_size() / get_y_size();
for (int xt = 0; xt < get_x_size(); xt++) {
int xs = xt * copy.get_x_size() / get_x_size();
set_xel(xt, yt, copy.get_xel(xs, ys));
}
}
if (has_alpha() && copy.has_alpha()) {
for (int yt = 0; yt < get_y_size(); yt++) {
int ys = yt * copy.get_y_size() / get_y_size();
for (int xt = 0; xt < get_x_size(); xt++) {
int xs = xt * copy.get_x_size() / get_x_size();
set_alpha(xt, yt, copy.get_alpha(xs, ys));
}
}
}
}
////////////////////////////////////////////////////////////////////
// Function: PNMImage::make_histogram
// Access: Published

View File

@ -239,6 +239,7 @@ PUBLISHED:
INLINE void box_filter(double radius = 1.0);
INLINE void gaussian_filter(double radius = 1.0);
void unfiltered_stretch_from(const PNMImage &copy);
void box_filter_from(double radius, const PNMImage &copy);
void gaussian_filter_from(double radius, const PNMImage &copy);
void quick_filter_from(const PNMImage &copy,

View File

@ -54,7 +54,7 @@ private:
typedef pmap<Filename, PT(BamCacheRecord) > Records;
Records _records;
off_t _cache_size;
streamsize _cache_size;
// This structure is a temporary container. It is only filled in
// while reading from a bam file.

View File

@ -84,14 +84,14 @@ private:
Filename _source_pathname;
Filename _cache_filename;
time_t _recorded_time;
off_t _record_size; // this is accurate only in the index file.
streamsize _record_size; // this is accurate only in the index file.
time_t _source_timestamp; // Not record to the cache file.
class DependentFile {
public:
Filename _pathname;
time_t _timestamp;
off_t _size;
streamsize _size;
};
typedef pvector<DependentFile> DependentFiles;

View File

@ -141,18 +141,28 @@ get_datagram(Datagram &data) {
// First, get the size of the upcoming datagram.
StreamReader reader(_in, false);
size_t num_bytes = reader.get_uint32();
PN_uint32 num_bytes_32 = reader.get_uint32();
if (_in->fail() || _in->eof()) {
return false;
}
if (num_bytes == 0) {
if (num_bytes_32 == 0) {
// A special case for a zero-length datagram: no need to try to
// read any data.
data.clear();
return true;
}
streamsize num_bytes = (streamsize)num_bytes_32;
if (num_bytes_32 == (PN_uint32)-1) {
// Another special case for a value larger than 32 bits.
num_bytes = reader.get_uint64();
}
// Make sure we have a reasonable datagram size for putting into
// memory.
nassertr(num_bytes == (size_t)num_bytes, false);
// Now, read the datagram itself.
// If the number of bytes is large, we will need to allocate a
@ -209,11 +219,17 @@ save_datagram(SubfileInfo &info) {
// First, get the size of the upcoming datagram.
StreamReader reader(_in, false);
size_t num_bytes = reader.get_uint32();
size_t num_bytes_32 = reader.get_uint32();
if (_in->fail() || _in->eof()) {
return false;
}
streamsize num_bytes = (streamsize)num_bytes_32;
if (num_bytes_32 == (PN_uint32)-1) {
// Another special case for a value larger than 32 bits.
num_bytes = reader.get_uint64();
}
// If this stream is file-based, we can just point the SubfileInfo
// directly into this file.
if (_file != (FileReference *)NULL) {
@ -238,12 +254,12 @@ save_datagram(SubfileInfo &info) {
<< "Copying " << num_bytes << " bytes to " << tfile->get_filename() << "\n";
}
size_t num_remaining = num_bytes;
streamsize num_remaining = num_bytes;
static const size_t buffer_size = 4096;
char buffer[buffer_size];
_in->read(buffer, min(buffer_size, num_remaining));
size_t count = _in->gcount();
_in->read(buffer, min((streamsize)buffer_size, num_remaining));
streamsize count = _in->gcount();
while (count != 0) {
out.write(buffer, count);
if (out.fail()) {
@ -256,7 +272,7 @@ save_datagram(SubfileInfo &info) {
if (num_remaining == 0) {
break;
}
_in->read(buffer, min(buffer_size, num_remaining));
_in->read(buffer, min((streamsize)buffer_size, num_remaining));
count = _in->gcount();
}

View File

@ -122,7 +122,15 @@ put_datagram(const Datagram &data) {
// First, write the size of the upcoming datagram.
StreamWriter writer(_out, false);
writer.add_uint32(data.get_length());
size_t num_bytes = data.get_length();
if (num_bytes == (PN_uint32)-1 || num_bytes != (PN_uint32)num_bytes) {
// Write a large value as a 64-bit size.
writer.add_uint32((PN_uint32)-1);
writer.add_uint64(num_bytes);
} else {
// Write a value that fits in 32 bits.
writer.add_uint32((PN_uint32)num_bytes);
}
// Now, write the datagram itself.
_out->write((const char *)data.get_data(), data.get_length());
@ -158,19 +166,25 @@ copy_datagram(SubfileInfo &result, const Filename &filename) {
return false;
}
off_t size = vfile->get_file_size(in);
size_t num_remaining = (size_t)size;
nassertr(num_remaining == size, false);
streamsize size = vfile->get_file_size(in);
streamsize num_remaining = size;
StreamWriter writer(_out, false);
writer.add_uint32(num_remaining);
if (num_remaining == (PN_uint32)-1 || num_remaining != (PN_uint32)num_remaining) {
// Write a large value as a 64-bit size.
writer.add_uint32((PN_uint32)-1);
writer.add_uint64(num_remaining);
} else {
// Write a value that fits in 32 bits.
writer.add_uint32((PN_uint32)num_remaining);
}
static const size_t buffer_size = 4096;
char buffer[buffer_size];
streampos start = _out->tellp();
in->read(buffer, min(buffer_size, num_remaining));
size_t count = in->gcount();
streamsize count = in->gcount();
while (count != 0) {
_out->write(buffer, count);
if (_out->fail()) {
@ -220,10 +234,17 @@ copy_datagram(SubfileInfo &result, const SubfileInfo &source) {
return false;
}
size_t num_remaining = source.get_size();
streamsize num_remaining = source.get_size();
StreamWriter writer(_out, false);
writer.add_uint32(num_remaining);
if (num_remaining == (PN_uint32)-1 || num_remaining != (PN_uint32)num_remaining) {
// Write a large value as a 64-bit size.
writer.add_uint32((PN_uint32)-1);
writer.add_uint64(num_remaining);
} else {
// Write a value that fits in 32 bits.
writer.add_uint32((PN_uint32)num_remaining);
}
static const size_t buffer_size = 4096;
char buffer[buffer_size];
@ -231,7 +252,7 @@ copy_datagram(SubfileInfo &result, const SubfileInfo &source) {
streampos start = _out->tellp();
in.seekg(source.get_start());
in.read(buffer, min(buffer_size, num_remaining));
size_t count = in.gcount();
streamsize count = in.gcount();
while (count != 0) {
_out->write(buffer, count);
if (_out->fail()) {