From c3dbc8fbedcbdb0752a4766371ae257c7eb63f6b Mon Sep 17 00:00:00 2001 From: Mike Goslin Date: Wed, 13 Dec 2000 20:39:21 +0000 Subject: [PATCH] *** empty log message *** --- panda/src/downloader/Sources.pp | 3 +- panda/src/downloader/download.I | 90 +++++ panda/src/downloader/download.cxx | 649 ++++++++++++++++++++++++++++++ panda/src/downloader/download.h | 119 ++++++ 4 files changed, 860 insertions(+), 1 deletion(-) create mode 100644 panda/src/downloader/download.I create mode 100644 panda/src/downloader/download.cxx create mode 100644 panda/src/downloader/download.h diff --git a/panda/src/downloader/Sources.pp b/panda/src/downloader/Sources.pp index ea90ce9ad1..ea831c0563 100644 --- a/panda/src/downloader/Sources.pp +++ b/panda/src/downloader/Sources.pp @@ -17,7 +17,8 @@ multiplexStream.I multiplexStream.cxx multiplexStream.h \ multiplexStreamBuf.I multiplexStreamBuf.cxx multiplexStreamBuf.h \ patcher.cxx \ - patcher.h + patcher.h \ + download.I download.cxx download.h #define IF_ZLIB_SOURCES \ decompressor.cxx decompressor.h zcompressor.I zcompressor.cxx \ diff --git a/panda/src/downloader/download.I b/panda/src/downloader/download.I new file mode 100644 index 0000000000..ebf09af57e --- /dev/null +++ b/panda/src/downloader/download.I @@ -0,0 +1,90 @@ +// Filename: download.I +// Created by: mike (09Jan97) +// +//////////////////////////////////////////////////////////////////// + +#include "config_downloader.h" + +//////////////////////////////////////////////////////////////////// +// Function: Download::set_frequency +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +INLINE void Download:: +set_frequency(float frequency) { + nassertv(frequency > 0.0); + if (_frequency != frequency) { + _frequency = frequency; + _recompute_buffer = true; + } +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::get_frequency +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +INLINE float Download:: +get_frequency(void) const { + return _frequency; +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::set_byte_rate +// Access: Public +// Description: Note: modem speeds are reported in bits, so you +// need to convert! +//////////////////////////////////////////////////////////////////// +INLINE void Download:: +set_byte_rate(float bytes) { + nassertv(bytes > 0.0); + if (_byte_rate != bytes) { + _byte_rate = bytes; + _recompute_buffer = true; + } +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::get_byte_rate +// Access: Public +// Description: Returns byte rate in bytes. +//////////////////////////////////////////////////////////////////// +INLINE float Download:: +get_byte_rate(void) const { + return _byte_rate; +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::set_disk_write_frequency +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +INLINE void Download:: +set_disk_write_frequency(int frequency) { + nassertv(frequency > 0); + if (_disk_write_frequency != frequency) { + _disk_write_frequency = frequency; + _recompute_buffer = true; + } +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::get_disk_write_frequency +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +INLINE int Download:: +get_disk_write_frequency(void) const { + return _disk_write_frequency; +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::get_bytes_per_second +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +INLINE float Download:: +get_bytes_per_second(void) const { + nassertr(_tlast - _tfirst > 0.0, 0.0); + return (float)((double)_current_status->_total_bytes / (_tlast - _tfirst)); +} diff --git a/panda/src/downloader/download.cxx b/panda/src/downloader/download.cxx new file mode 100644 index 0000000000..4e9fe3930c --- /dev/null +++ b/panda/src/downloader/download.cxx @@ -0,0 +1,649 @@ +// Filename: download.cxx +// Created by: mike (09Jan97) +// +//////////////////////////////////////////////////////////////////// +// +//////////////////////////////////////////////////////////////////// +// Includes +//////////////////////////////////////////////////////////////////// +#include "download.h" +#include "config_downloader.h" + +#include +#include +#include + +#if !defined(WIN32_VC) + #include + #include + #include + #include +#endif + +//////////////////////////////////////////////////////////////////// +// Defines +//////////////////////////////////////////////////////////////////// +enum SafeSendCode { + SS_success = 1, + SS_error = -1, + SS_timeout = -2, +}; + +enum FastReceiveCode { + FR_eof = 2, + FR_success = 1, + FR_error = -1, + FR_timeout = -2, + FR_no_data = -3, +}; + +//////////////////////////////////////////////////////////////////// +// Function: Download::Constructor +// Access: Published +// Description: +//////////////////////////////////////////////////////////////////// +Download:: +Download(void) { + _frequency = downloader_frequency; + _byte_rate = downloader_byte_rate; + _disk_write_frequency = downloader_disk_write_frequency; + nassertv(_frequency > 0 && _byte_rate > 0 && _disk_write_frequency > 0); + _receive_size = _byte_rate * _frequency; + _disk_buffer_size = _disk_write_frequency * _receive_size; + _buffer = new Buffer(_disk_buffer_size); + + _connected = false; + // We need to flush after every write in case we're interrupted + _dest_stream.setf(ios::unitbuf, 0); + _current_status = NULL; + _recompute_buffer = false; + + _tfirst = 0.0; + _tlast = 0.0; + _got_any_data = false; + +#if defined(WIN32) + WSAData mydata; + int answer1 = WSAStartup(0x0101, &mydata); + if(answer1 != 0) { + downloader_cat.error() + << "Downloader::Downloader() - Error initializing TCP stack!" + << endl; + } +#endif +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::Destructor +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +Download:: +~Download() { + if (_connected) + disconnect_from_server(); + delete _buffer; + if (_current_status != NULL) + delete _current_status; +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::connect_to_server +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +bool Download:: +connect_to_server(const string &name, uint port) { + if (downloader_cat.is_debug()) + downloader_cat.debug() + << "Download connecting to server: " << name << " on port: " + << port << endl; + + _server_name = name; + + _sin.sin_family = PF_INET; + _sin.sin_port = htons(port); + ulong addr = (ulong)inet_addr(name.c_str()); + struct hostent *hp = NULL; + + if (addr == INADDR_NONE) { + hp = gethostbyname(name.c_str()); + if (hp != NULL) + (void)memcpy(&_sin.sin_addr, hp->h_addr, (uint)hp->h_length); + else { + downloader_cat.error() + << "Downloader::connect_to_server() - gethostbyname() failed: " + << strerror(errno) << endl; + return false; + } + } else + (void)memcpy(&_sin.sin_addr, &addr, sizeof(addr)); + + return connect_to_server(); +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::connect_to_server +// Access: Private +// Description: +//////////////////////////////////////////////////////////////////// +bool Download:: +connect_to_server(void) { + if (_connected == true) + return true; + + _socket = 0xffffffff; + _socket = socket(PF_INET, SOCK_STREAM, 0); + if (_socket == (int)0xffffffff) { + downloader_cat.error() + << "Download::connect_to_server() - socket failed: " + << strerror(errno) << endl; + return false; + } + + _connected = true; + + if (connect(_socket, (struct sockaddr *)&_sin, sizeof(_sin)) < 0) { + downloader_cat.error() + << "Download::connect_to_server() - connect() failed: " + << strerror(errno) << endl; + disconnect_from_server(); + _connected = false; + } + + return _connected; +} + +/////////////////////////////////////////////////////////////////// +// Function: Download::disconnect_from_server +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +void Download:: +disconnect_from_server(void) { + if (downloader_cat.is_debug()) + downloader_cat.debug() + << "Download disconnecting from server..." << endl; +#if defined(WIN32) + (void)closesocket(_socket); +#else + (void)close(_socket); +#endif + _connected = false; +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::safe_send +// Access: Private +// Description: +//////////////////////////////////////////////////////////////////// +int Download:: +safe_send(int socket, const char *data, int length, long timeout) { + if (length == 0) { + downloader_cat.error() + << "Download::safe_send() - requested 0 length send!" << endl; + return SS_error; + } + int bytes = 0; + struct timeval tv; + tv.tv_sec = timeout; + tv.tv_usec = 0; + fd_set wset; + FD_ZERO(&wset); + while (bytes < length) { + FD_SET(socket, &wset); + int sret = select(socket + 1, NULL, &wset, NULL, &tv); + if (sret == 0) { + downloader_cat.error() + << "Download::safe_send() - select timed out after: " + << timeout << " seconds" << endl; + return SS_timeout; + } else if (sret == -1) { + downloader_cat.error() + << "Download::safe_send() - error: " << strerror(errno) << endl; + return SS_error; + } + int ret = send(socket, data, length, 0); + if (ret > 0) + bytes += ret; + else { + downloader_cat.error() + << "Download::safe_send() - error: " << strerror(errno) << endl; + return SS_error; + } + } + return SS_success; +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::fast_receive +// Access: Private +// Description: +//////////////////////////////////////////////////////////////////// +int Download:: +fast_receive(int socket, DownloadStatus *status, int rec_size) { + nassertr(status != NULL, FR_error); + if (rec_size <= 0) { + downloader_cat.error() + << "Download::fast_receive() - Invalid receive size: " << rec_size + << endl; + return FR_error; + } + + // Poll the socket with select() to see if there is any data + struct timeval tv; + tv.tv_sec = 0; + tv.tv_usec = 0; + fd_set rset; + FD_ZERO(&rset); + FD_SET(socket, &rset); + int sret = select(socket + 1, &rset, NULL, NULL, &tv); + if (sret == 0) { + return FR_no_data; + } else if (sret == -1) { + downloader_cat.error() + << "Downloader::safe_receive() - error: " << strerror(errno) << endl; + return FR_error; + } + int ret = recv(socket, status->_next_in, rec_size, 0); + if (ret == 0) { + return FR_eof; + } else if (ret == -1) { + downloader_cat.error() + << "Download::fast_receive() - error: " << strerror(errno) << endl; + return FR_error; + } + if (downloader_cat.is_debug()) + downloader_cat.debug() + << "Download::fast_receive() - recv() got: " << ret << " bytes" + << endl; + status->_next_in += ret; + status->_bytes_in_buffer += ret; + status->_total_bytes += ret; + return ret; +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::initiate +// Access: Published +// Description: Initiate the download of a complete file from the server. +//////////////////////////////////////////////////////////////////// +int Download:: +initiate(const string &file_name, Filename file_dest) { + return initiate(file_name, file_dest, 0, 0, 0, false); +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::initiate +// Access: Published +// Description: Initiate the download of a file from a server. +//////////////////////////////////////////////////////////////////// +int Download:: +initiate(const string &file_name, Filename file_dest, + int first_byte, int last_byte, int total_bytes, + bool partial_content) { + + // Connect to the server + if (connect_to_server() == false) + return DS_error_connect; + + // Attempt to open the destination file + file_dest.set_binary(); + bool result; + if (partial_content == true && first_byte > 0) + result = file_dest.open_append(_dest_stream); + else + result = file_dest.open_write(_dest_stream); + if (result == false) { + downloader_cat.error() + << "Downloader::download() - Error opening file: " << file_dest + << " for writing" << endl; + return DS_error_write; + } + + // Send an HTTP request for the file to the server + string request = "GET "; + request += file_name; + request += " HTTP/1.1\012Host: "; + request += _server_name; + request += "\012Connection: close"; + if (partial_content == true) { + if (downloader_cat.is_debug()) + downloader_cat.debug() + << "Downloader::download() - Requesting byte range: " << first_byte + << "-" << last_byte << endl; + request += "\012Range: bytes="; + stringstream start_stream; + start_stream << first_byte << "-" << last_byte; + request += start_stream.str(); + } + request += "\012\012"; + int outlen = request.size(); + if (downloader_cat.is_debug()) + downloader_cat.debug() + << "Downloader::download() - Sending request:\n" << request << endl; + int send_ret = safe_send(_socket, request.c_str(), outlen, + (long)downloader_timeout); + + // Handle timeouts on the send + if (send_ret == SS_timeout) { + downloader_cat.error() + << "Download::initiate() - send timed out" << endl; + return DS_timeout; + } + + if (send_ret == SS_error) + return DS_error_connect; + + // Create a download status to maintain download progress information + if (_current_status != NULL) + delete _current_status; + _current_status = new DownloadStatus(_buffer->_buffer, + first_byte, last_byte, total_bytes, + partial_content); + + return DS_success; +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::run +// Access: Published +// Description: +//////////////////////////////////////////////////////////////////// +int Download:: +run(void) { + if (_current_status == NULL) { + downloader_cat.error() + << "Download::run() - Did not call initiate() first" << endl; + return DS_error; + } + + if (connect_to_server() == false) + return DS_error_connect; + + double t0 = _clock.get_real_time(); + if (_tfirst == 0.0) { + _tfirst = t0; + } + if (t0 - _tlast < _frequency) + return DS_ok; + + // Recompute the buffer size if necessary + if (_recompute_buffer == true) { + + // Flush the current buffer if it holds any data + if (_current_status->_bytes_in_buffer > 0) { + if (write_to_disk(_current_status) == false) { + return DS_error_write; + } + } + + // Allocate a new buffer + _buffer.clear(); + _receive_size = (int)ceil(_frequency * _byte_rate); + _disk_buffer_size = _receive_size * _disk_write_frequency; + _buffer = new Buffer(_disk_buffer_size); + _current_status->_buffer = _buffer->_buffer; + _current_status->reset(); + + } else if (_current_status->_bytes_in_buffer + _receive_size > + _disk_buffer_size) { + + // Flush the current buffer if the next request would overflow it + if (downloader_cat.is_debug()) + downloader_cat.debug() + << "Download::run() - Flushing buffer" << endl; + if (write_to_disk(_current_status) == false) + return DS_error_write; + } + + // Attempt to receive the bytes from the socket + int bytes_read = fast_receive(_socket, _current_status, _receive_size); + _tlast = _clock.get_real_time(); + + // Check for end of file + if (bytes_read == 0) { + if (_got_any_data == true) { + if (_current_status->_bytes_in_buffer > 0) { + if (write_to_disk(_current_status) == false) + return DS_error_write; + } + return DS_success; + } else { + if (downloader_cat.is_debug()) + downloader_cat.debug() + << "Download::run() - Got 0 bytes" << endl; + return DS_ok; + } + } else if (bytes_read < 0) { + return DS_error; + } + + _got_any_data = true; + return DS_ok; +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::parse_http_response +// Access: Private +// Description: Check the HTTP response from the server +//////////////////////////////////////////////////////////////////// +bool Download:: +parse_http_response(const string &resp) { + size_t ws = resp.find(" ", 0); + string httpstr = resp.substr(0, ws); + if (!(httpstr == "HTTP/1.1")) { + downloader_cat.error() + << "Download::parse_http_response() - not HTTP/1.1 - got: " + << httpstr << endl; + return false; + } + size_t ws2 = resp.find(" ", ws); + string numstr = resp.substr(ws, ws2); + nassertr(numstr.length() > 0, false); + int num = atoi(numstr.c_str()); + switch (num) { + case 200: + case 206: + return true; + case 202: + // Accepted - server may not honor request, though + if (downloader_cat.is_debug()) + downloader_cat.debug() + << "Download::parse_http_response() - got a 202 Accepted - " + << "server does not guarantee to honor this request" << endl; + return true; + case 201: + case 203: + case 204: + case 205: + default: + break; + } + + downloader_cat.error() + << "Download::parse_http_response() - Invalid response: " + << resp << endl; + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::parse_header +// Access: Private +// Description: Looks for a valid header. If it finds one, it +// calculates the header length and strips it from +// the download status structure. Function returns false +// on an error condition, otherwise true. +//////////////////////////////////////////////////////////////////// +bool Download:: +parse_header(DownloadStatus *status) { + nassertr(status != NULL, false); + + if (status->_header_is_complete == true) + return true; + + if (status->_bytes_in_buffer == 0) { + downloader_cat.error() + << "Download::parse_header() - Empty buffer!" << endl; + return false; + } + + string bufstr((char *)status->_start, status->_bytes_in_buffer); + size_t p = 0; + while (p < bufstr.length()) { + // Server sends out CR LF (\r\n) as newline delimiter + size_t nl = bufstr.find("\015\012", p); + if (nl == string::npos) { + downloader_cat.error() + << "Download::parse_header() - No newlines in buffer of " + << "length: " << status->_bytes_in_buffer << endl; + return false; + } else if (p == 0 && nl == p) { + downloader_cat.error() + << "Download::parse_header() - Buffer begins with newline!" + << endl; + return false; + } + + string component = bufstr.substr(p, nl - p); + + // The first line of the response should say whether + // got an error or not + if (status->_first_line_complete == false) { + status->_first_line_complete = true; + if (parse_http_response(component) == true) { + if (downloader_cat.is_debug()) + downloader_cat.debug() + << "Download::parse_header() - Header is valid: " + << component << endl; + status->_header_is_valid = true; + } else { + return false; + } + } + + // Look for content length + size_t cpos = component.find(":"); + string tline = component.substr(0, cpos); + if (status->_partial_content == true && tline == "Content-Length") { + tline = component.substr(cpos + 2, string::npos); + int server_download_bytes = atoi(tline.c_str()); + int client_download_bytes = status->_last_byte - status->_first_byte; + if (status->_first_byte == 0) + client_download_bytes += 1; + if (client_download_bytes != server_download_bytes) { + downloader_cat.error() + << "Download::parse_header() - server size = " + << server_download_bytes << ", client size = " + << client_download_bytes << " (" + << status->_last_byte << "-" << status->_first_byte << ")" << endl; + return false; + } + } + + // Two consecutive (CR LF)s indicates end of HTTP header + if (nl == p) { + if (downloader_cat.is_debug()) + downloader_cat.debug() + << "Download::parse_header() - Header is complete" << endl; + status->_header_is_complete = true; + + // Strip the header out of the status buffer + int header_length = nl + 2; + status->_start += header_length; + status->_bytes_in_buffer -= header_length; + + if (downloader_cat.is_debug()) + downloader_cat.debug() + << "Download::parse_header() - Stripping out header of size: " + << header_length << endl; + + return true; + } + + p = nl + 2; + } + + if (status->_header_is_complete == false) { + if (downloader_cat.is_debug()) + downloader_cat.debug() + << "Download::parse_header() - Reached end of buffer without " + << "successfully parsing the header - buffer size: " + << status->_bytes_in_buffer << endl; + } + + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::write_to_disk +// Access: Private +// Description: Writes a download to disk. If there is a header, +// the pointer and size are adjusted so the header +// is excluded. Function returns false on error +// condition. +//////////////////////////////////////////////////////////////////// +bool Download:: +write_to_disk(DownloadStatus *status) { + nassertr(status != NULL, false); + + // Ensure the header has been parsed successfully first + if (parse_header(status) == false) + return false; + + if (status->_header_is_complete == false) { + downloader_cat.error() + << "Download::write_to_disk() - Incomplete HTTP header - " + << "(or header was larger than download buffer) - " + << "try increasing download-buffer-size" << endl; + return false; + } + + // Write what we have so far to disk + if (status->_bytes_in_buffer > 0) { + if (downloader_cat.is_debug()) + downloader_cat.debug() + << "Download::write_to_disk() - Writing " + << status->_bytes_in_buffer << " to disk" << endl; + + _dest_stream.write(status->_start, status->_bytes_in_buffer); + status->_total_bytes_written += status->_bytes_in_buffer; + } + + status->reset(); + + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::DownloadStatus::constructor +// Access: Private +// Description: +//////////////////////////////////////////////////////////////////// +Download::DownloadStatus:: +DownloadStatus(char *buffer, int first_byte, int last_byte, + int total_bytes, bool partial_content) { + _first_line_complete = false; + _header_is_complete = false; + _header_is_valid = false; + _buffer = buffer; + _first_byte = first_byte; + _last_byte = last_byte; + _total_bytes = total_bytes; + _partial_content = partial_content; + reset(); +} + +//////////////////////////////////////////////////////////////////// +// Function: Download::DownloadStatus::reset +// Access: Public +// Description: Resets the status buffer for more downloading after +// a write. +//////////////////////////////////////////////////////////////////// +void Download::DownloadStatus:: +reset(void) { + _start = _buffer; + _next_in = _start; + _bytes_in_buffer = 0; + _total_bytes_written = 0; +} diff --git a/panda/src/downloader/download.h b/panda/src/downloader/download.h new file mode 100644 index 0000000000..f663e261ec --- /dev/null +++ b/panda/src/downloader/download.h @@ -0,0 +1,119 @@ +// Filename: download.h +// Created by: mike (09Jan97) +// +//////////////////////////////////////////////////////////////////// +// +#ifndef DOWNLOAD_H +#define DOWNLOAD_H +// +//////////////////////////////////////////////////////////////////// +// Includes +//////////////////////////////////////////////////////////////////// +#include +#include +#include +#include +#include +#include + +#if defined(WIN32_VC) + #include +#else + #include + #include + #include +#endif + +//////////////////////////////////////////////////////////////////// +// Class : Download +// Description : +//////////////////////////////////////////////////////////////////// +class EXPCL_PANDAEXPRESS Download { +PUBLISHED: + enum DownloadCode { + DS_ok = 3, + DS_write = 2, + DS_success = 1, + DS_error = -1, + DS_timeout = -2, + DS_error_write = -3, + DS_error_connect = -4, + }; + + Download(void); + virtual ~Download(void); + + bool connect_to_server(const string &name, uint port=80); + void disconnect_from_server(void); + + int initiate(const string &file_name, Filename file_dest); + int initiate(const string &file_name, Filename file_dest, + int first_byte, int last_byte, int total_bytes, + bool partial_content = true); + int run(void); + + INLINE void set_frequency(float frequency); + INLINE float get_frequency(void) const; + INLINE void set_byte_rate(float bytes); + INLINE float get_byte_rate(void) const; + INLINE void set_disk_write_frequency(int frequency); + INLINE int get_disk_write_frequency(void) const; + INLINE float get_bytes_per_second(void) const; + +private: + class DownloadStatus { + public: + DownloadStatus(char *buffer, int first_byte, int last_byte, + int total_bytes, bool partial_content); + void reset(void); + + public: + bool _first_line_complete; + bool _header_is_complete; + bool _header_is_valid; + char *_start; + char *_next_in; + int _bytes_in_buffer; + int _total_bytes_written; + int _first_byte; + int _last_byte; + int _total_bytes; + bool _partial_content; + char *_buffer; + }; + + INLINE void recompute_buffer(void); + + bool connect_to_server(void); + int safe_send(int socket, const char *data, int length, long timeout); + int fast_receive(int socket, DownloadStatus *status, int rec_size); + bool parse_http_response(const string &resp); + bool parse_header(DownloadStatus *status); + bool write_to_disk(DownloadStatus *status); + +private: + bool _connected; + int _socket; + string _server_name; + struct sockaddr_in _sin; + + PT(Buffer) _buffer; + int _disk_write_frequency; + float _frequency; + float _byte_rate; + int _receive_size; + int _disk_buffer_size; + ofstream _dest_stream; + bool _recompute_buffer; + + DownloadStatus *_current_status; + bool _got_any_data; + + double _tlast; + double _tfirst; + ClockObject _clock; +}; + +#include "download.I" + +#endif