diff --git a/panda/src/downloader/bioPtr.I b/panda/src/downloader/bioPtr.I index d8414fc312..935e3f15d9 100644 --- a/panda/src/downloader/bioPtr.I +++ b/panda/src/downloader/bioPtr.I @@ -75,3 +75,25 @@ INLINE BIO *BioPtr:: get_bio() const { return _bio; } + +//////////////////////////////////////////////////////////////////// +// Function: BioPtr::get_server_name +// Access: Public +// Description: Returns the name of the server we are (or should be) +// connected to. +//////////////////////////////////////////////////////////////////// +INLINE const string &BioPtr:: +get_server_name() const { + return _server_name; +} + +//////////////////////////////////////////////////////////////////// +// Function: BioPtr::get_port +// Access: Public +// Description: Returns the port on which we are (or should be) +// connected. +//////////////////////////////////////////////////////////////////// +INLINE int BioPtr:: +get_port() const { + return _port; +} diff --git a/panda/src/downloader/bioPtr.h b/panda/src/downloader/bioPtr.h index 299cab767a..8bf9352f5d 100644 --- a/panda/src/downloader/bioPtr.h +++ b/panda/src/downloader/bioPtr.h @@ -52,6 +52,8 @@ public: INLINE BIO *get_bio() const; bool connect() const; + INLINE const string &get_server_name() const; + INLINE int get_port() const; private: BIO *_bio; diff --git a/panda/src/downloader/bioStreamBuf.cxx b/panda/src/downloader/bioStreamBuf.cxx index bac868129a..b051e8fd6e 100644 --- a/panda/src/downloader/bioStreamBuf.cxx +++ b/panda/src/downloader/bioStreamBuf.cxx @@ -103,6 +103,7 @@ underflow() { // Oops, we didn't read what we thought we would. if (read_count <= 0) { _is_closed = !BIO_should_retry(*_source); + gbump(num_bytes); return EOF; } diff --git a/panda/src/downloader/chunkedStreamBuf.cxx b/panda/src/downloader/chunkedStreamBuf.cxx index 50b87a8055..fb3cdd1cac 100644 --- a/panda/src/downloader/chunkedStreamBuf.cxx +++ b/panda/src/downloader/chunkedStreamBuf.cxx @@ -116,6 +116,7 @@ underflow() { if (read_count != num_bytes) { // Oops, we didn't read what we thought we would. if (read_count == 0) { + gbump(num_bytes); return EOF; } @@ -146,16 +147,16 @@ read_chars(char *start, size_t length) { // Extract some of the bytes remaining in the chunk. length = min(length, _chunk_remaining); (*_source)->read(start, length); - length = (*_source)->gcount(); - _chunk_remaining -= length; - return length; + size_t read_count = (*_source)->gcount(); + _chunk_remaining -= read_count; + return read_count; } // Read the next chunk. string line; - getline(**_source, line); - if (!line.empty() && line[line.length() - 1] == '\r') { - line = line.substr(0, line.length() - 1); + if (!http_getline(line)) { + // EOF (or data unavailable) while trying to read the chunk size. + return 0; } size_t chunk_size = (size_t)strtol(line.c_str(), NULL, 16); if (chunk_size == 0) { @@ -175,4 +176,40 @@ read_chars(char *start, size_t length) { return read_chars(start, length); } +//////////////////////////////////////////////////////////////////// +// Function: ChunkedStreamBuf::http_getline +// Access: Private +// Description: Reads a single line from the stream. Returns +// true if the line is successfully retrieved, or false +// if a complete line has not yet been received or if +// the connection has been closed. +//////////////////////////////////////////////////////////////////// +bool ChunkedStreamBuf:: +http_getline(string &str) { + nassertr(!_source.is_null(), false); + int ch = (*_source)->get(); + while (!(*_source)->eof() && !(*_source)->fail()) { + switch (ch) { + case '\n': + // end-of-line character, we're done. + if (downloader_cat.is_spam()) { + downloader_cat.spam() << "recv: " << _working_getline << "\n"; + } + str = _working_getline; + _working_getline = string(); + return true; + + case '\r': + // Ignore CR characters. + break; + + default: + _working_getline += (char)ch; + } + ch = (*_source)->get(); + } + + return false; +} + #endif // HAVE_SSL diff --git a/panda/src/downloader/chunkedStreamBuf.h b/panda/src/downloader/chunkedStreamBuf.h index 5756adc5c0..bfe4bf91d7 100644 --- a/panda/src/downloader/chunkedStreamBuf.h +++ b/panda/src/downloader/chunkedStreamBuf.h @@ -47,10 +47,12 @@ protected: private: size_t read_chars(char *start, size_t length); + bool http_getline(string &str); PT(BioStreamPtr) _source; size_t _chunk_remaining; bool _done; + string _working_getline; PT(HTTPChannel) _doc; int _read_index; diff --git a/panda/src/downloader/httpChannel.I b/panda/src/downloader/httpChannel.I index 58820ee7d8..cf8d76e836 100644 --- a/panda/src/downloader/httpChannel.I +++ b/panda/src/downloader/httpChannel.I @@ -26,7 +26,8 @@ //////////////////////////////////////////////////////////////////// INLINE bool HTTPChannel:: is_valid() const { - return (!_source.is_null() && (_status_code / 100) == 2); + return (!_source.is_null() && _state != S_failure && + (_status_code / 100) == 2); } //////////////////////////////////////////////////////////////////// @@ -192,7 +193,9 @@ get_file_size() const { //////////////////////////////////////////////////////////////////// INLINE bool HTTPChannel:: post_form(const URLSpec &url, const string &body) { - return send_request("POST", url, body); + begin_request("POST", url, body, false); + run(); + return is_valid(); } //////////////////////////////////////////////////////////////////// @@ -203,7 +206,9 @@ post_form(const URLSpec &url, const string &body) { //////////////////////////////////////////////////////////////////// INLINE bool HTTPChannel:: get_document(const URLSpec &url) { - return send_request("GET", url, string()); + begin_request("GET", url, string(), false); + run(); + return is_valid(); } //////////////////////////////////////////////////////////////////// @@ -217,5 +222,75 @@ get_document(const URLSpec &url) { //////////////////////////////////////////////////////////////////// INLINE bool HTTPChannel:: get_header(const URLSpec &url) { - return send_request("HEAD", url, string()); + begin_request("HEAD", url, string(), false); + run(); + return is_valid(); +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::request_post_form +// Access: Published +// Description: Posts form data to a particular URL and retrieves the +// response, all using non-blocking I/O. See +// request_document() and post_form(). +// +// It is important to note that you *must* call run() +// repeatedly after calling this method until run() +// returns false, and you may not call any other +// document posting or retrieving methods using the +// HTTPChannel object in the interim, or your form data +// may not get posted. +//////////////////////////////////////////////////////////////////// +INLINE void HTTPChannel:: +request_post_form(const URLSpec &url, const string &body) { + begin_request("POST", url, body, true); +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::request_document +// Access: Published +// Description: Begins a non-blocking request to retrieve a given +// document. This method will return immediately, even +// before a connection to the server has necessarily +// been established; you must then call run() from time +// to time until the return value of run() is false. +// Then you may check is_valid() and get_status_code() +// to determine the status of your request. +// +// If a previous request had been pending, that request +// is discarded. +//////////////////////////////////////////////////////////////////// +INLINE void HTTPChannel:: +request_document(const URLSpec &url) { + begin_request("GET", url, string(), true); +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::request_header +// Access: Published +// Description: Begins a non-blocking request to retrieve a given +// header. See request_document() and get_header(). +//////////////////////////////////////////////////////////////////// +INLINE void HTTPChannel:: +request_header(const URLSpec &url) { + begin_request("HEAD", url, string(), true); +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::check_socket +// Access: Private +// Description: Checks whether the connection to the server has been +// closed after a failed read. If it has, issues a +// warning and calls free_bio(). +//////////////////////////////////////////////////////////////////// +INLINE void HTTPChannel:: +check_socket() { + nassertv(!_source.is_null()); + if ((*_source)->is_closed()) { + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "Lost connection to server unexpectedly during read.\n"; + } + free_bio(); + } } diff --git a/panda/src/downloader/httpChannel.cxx b/panda/src/downloader/httpChannel.cxx index 62d73447ac..33eda15713 100644 --- a/panda/src/downloader/httpChannel.cxx +++ b/panda/src/downloader/httpChannel.cxx @@ -47,129 +47,20 @@ HTTPChannel(HTTPClient *client) : _client(client) { _persistent_connection = false; - _state = S_new; + _nonblocking = false; _read_index = 0; _file_size = 0; _status_code = 0; - _status_string = "No connection"; + _status_string = string(); _proxy = _client->get_proxy(); _http_version = _client->get_http_version(); _http_version_string = _client->get_http_version_string(); -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::send_request -// Access: Private -// Description: This is normally called immediately after -// construction to send the request to the server and -// read the response. It can't be called as part of the -// constructor because it may involve an up-and-down -// change in the reference count of the HTTPChannel -// object, which would inadvertently cause the object to -// be deleted if it hasn't returned from its constructor -// yet! -//////////////////////////////////////////////////////////////////// -bool HTTPChannel:: -send_request(const string &method, const URLSpec &url, const string &body) { - // Let's call this before we call make_header(), so we'll get the - // right HTTP version and proxy information etc. - set_url(url); - if (!prepare_for_next(true)) { - return false; - } - - string header; - make_header(header, method, url, body); - send_request(header, body, true); - - if ((get_status_code() / 100) == 3 && get_status_code() != 305) { - // Redirect. Should we handle it automatically? - if (!get_redirect().empty() && (method == "GET" || method == "HEAD")) { - // Sure! - pset already_seen; - bool keep_going; - do { - keep_going = false; - if (downloader_cat.is_debug()) { - downloader_cat.debug() - << "following redirect to " << get_redirect() << "\n"; - } - URLSpec new_url = get_redirect(); - if (already_seen.insert(new_url).second) { - if (url.has_username()) { - new_url.set_username(url.get_username()); - } - set_url(new_url); - if (prepare_for_next(true)) { - make_header(header, method, new_url, body); - send_request(header, body, true); - keep_going = - ((get_status_code() / 100) == 3 && get_status_code() != 305); - } - } - } while (keep_going); - } - } - - return is_valid(); -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::send_request -// Access: Private -// Description: This is a lower-level interface than the above -// send_request(); it accepts a header and body string -// that have already been defined. -//////////////////////////////////////////////////////////////////// -bool HTTPChannel:: -send_request(const string &header, const string &body, bool allow_reconnect) { - if (prepare_for_next(allow_reconnect)) { - // Tack on a proxy authorization if it is called for. Assume we - // can use the same authorization we used last time. - string proxy_auth_header = header; - if (!_proxy.empty() && !_client->_proxy_authorization.empty()) { - proxy_auth_header += "Proxy-Authorization: "; - proxy_auth_header += _client->_proxy_authorization; - proxy_auth_header += "\r\n"; - } - issue_request(proxy_auth_header, body); - - if (get_status_code() == 407 && !_proxy.empty()) { - // 407: not authorized to proxy. Try to get the authorization. - string authenticate_request = get_header_value("Proxy-Authenticate"); - string authorization; - if (get_authorization(authorization, authenticate_request, _proxy, true)) { - if (_client->_proxy_authorization != authorization) { - // Change the authorization. - _client->_proxy_authorization = authorization; - proxy_auth_header = header; - proxy_auth_header += "Proxy-Authorization: "; - proxy_auth_header += _client->_proxy_authorization; - proxy_auth_header += "\r\n"; - if (prepare_for_next(allow_reconnect)) { - issue_request(proxy_auth_header, body); - } - } - } - } - - if (get_status_code() == 401) { - // 401: not authorized to remote server. Try to get the authorization. - string authenticate_request = get_header_value("WWW-Authenticate"); - string authorization; - if (get_authorization(authorization, authenticate_request, _url, false)) { - string web_auth_header = proxy_auth_header; - web_auth_header += "Authorization: "; - web_auth_header += authorization; - web_auth_header += "\r\n"; - if (prepare_for_next(allow_reconnect)) { - issue_request(web_auth_header, body); - } - } - } - } - - return is_valid(); + _state = S_new; + _done_state = S_new; + _sent_so_far = 0; + _proxy_tunnel = false; + _body_stream = NULL; + _sbio = NULL; } //////////////////////////////////////////////////////////////////// @@ -284,6 +175,136 @@ write_headers(ostream &out) const { } } +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::run +// Access: Published +// Description: This must be called from time to time when +// non-blocking I/O is in use. It checks for data +// coming in on the socket and writes data out to the +// socket when possible, and does whatever processing is +// required towards completing the current task. +// +// The return value is true if the task is still pending +// (and run() will need to be called again in the +// future), or false if the current task is complete. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +run() { + if (downloader_cat.is_spam()) { + downloader_cat.spam() + << "begin run(), _state = " << (int)_state << ", _done_state = " + << (int)_done_state << "\n"; + } + if (_state == _done_state || _state == S_failure) { + return false; + } + + bool repeat_later; + do { + if (_bio.is_null()) { + // No connection. Attempt to establish one. + _proxy = _client->get_proxy(); + + if (_proxy.empty()) { + _bio = new BioPtr(_url); + } else { + _bio = new BioPtr(_proxy); + } + _source = new BioStreamPtr(new IBioStream(_bio)); + if (_nonblocking) { + BIO_set_nbio(*_bio, 1); + } + + _state = S_connecting; + } + + if (downloader_cat.is_spam()) { + downloader_cat.spam() + << "continue run(), _state = " << (int)_state << "\n"; + } + + switch (_state) { + case S_connecting: + repeat_later = run_connecting(); + break; + + case S_proxy_ready: + repeat_later = run_proxy_ready(); + break; + + case S_proxy_request_sent: + repeat_later = run_proxy_request_sent(); + break; + + case S_proxy_reading_header: + repeat_later = run_proxy_reading_header(); + break; + + case S_setup_ssl: + repeat_later = run_setup_ssl(); + break; + + case S_ssl_handshake: + repeat_later = run_ssl_handshake(); + break; + + case S_ready: + repeat_later = run_ready(); + break; + + case S_request_sent: + repeat_later = run_request_sent(); + break; + + case S_reading_header: + repeat_later = run_reading_header(); + break; + + case S_read_header: + repeat_later = run_read_header(); + break; + + case S_begin_body: + repeat_later = run_begin_body(); + break; + + case S_reading_body: + repeat_later = run_reading_body(); + break; + + case S_read_body: + repeat_later = run_read_body(); + break; + + case S_read_trailer: + repeat_later = run_read_trailer(); + break; + + default: + downloader_cat.warning() + << "Unhandled state " << (int)_state << "\n"; + return false; + } + + if (_state == _done_state || _state == S_failure) { + // We've reached our terminal state. + if (downloader_cat.is_spam()) { + downloader_cat.spam() + << "terminating run(), _state = " << (int)_state + << ", _done_state = " << (int)_done_state << "\n"; + } + return false; + } + } while (!repeat_later || _bio.is_null()); + + if (downloader_cat.is_spam()) { + downloader_cat.spam() + << "continue run() later, _state = " << (int)_state + << ", _done_state = " << (int)_done_state << "\n"; + } + return true; +} + //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::read_body // Access: Published @@ -292,7 +313,7 @@ write_headers(ostream &out) const { //////////////////////////////////////////////////////////////////// ISocketStream *HTTPChannel:: read_body() { - if (_state != S_read_header) { + if (_state != S_read_header && _state != S_begin_body) { return NULL; } @@ -305,7 +326,7 @@ read_body() { // the length of the file as we read it in chunks. The // IChunkedStream does this. _file_size = 0; - _state = S_started_body; + _state = S_reading_body; _read_index++; result = new IChunkedStream(_source, (HTTPChannel *)this); @@ -314,7 +335,7 @@ read_body() { // This is just the literal characters following the header, up // until _file_size bytes have been read (if content-length was // specified), or till end of file otherwise. - _state = S_started_body; + _state = S_reading_body; _read_index++; result = new IIdentityStream(_source, (HTTPChannel *)this, !content_length.empty(), _file_size); @@ -322,190 +343,186 @@ read_body() { return result; } - + //////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::establish_connection +// Function: HTTPChannel::run_connecting // Access: Private -// Description: Establishes a connection to the server, using the -// appropriate means. Returns true if a connection is -// successfully established (and _bio represents the -// connection), or false otherwise (and _bio is either -// NULL or an invalid connection.) +// Description: In this state, we have not yet established a +// network connection to the server (or proxy). //////////////////////////////////////////////////////////////////// bool HTTPChannel:: -establish_connection() { - bool result; - if (_proxy.empty()) { - if (_url.get_scheme() == "https") { - result = establish_https(); - } else { - result = establish_http(); +run_connecting() { + if (BIO_do_connect(*_bio) <= 0) { + if (BIO_should_retry(*_bio)) { + return true; } - } else { - if (_url.get_scheme() == "https") { - result = establish_https_proxy(); - } else { - result = establish_http_proxy(); - } - } - - return result; -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::establish_http -// Access: Private -// Description: Establishes a connection to the server directly, -// without using a proxy. -//////////////////////////////////////////////////////////////////// -bool HTTPChannel:: -establish_http() { - _bio = new BioPtr(_url); - _source = new BioStreamPtr(new IBioStream(_bio)); - return _bio->connect(); -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::establish_https -// Access: Private -// Description: Establishes a connection to the secure server -// directly, without using a proxy. -//////////////////////////////////////////////////////////////////// -bool HTTPChannel:: -establish_https() { - _bio = new BioPtr(_url); - _source = new BioStreamPtr(new IBioStream(_bio)); - if (!_bio->connect()) { - return false; - } - - return make_https_connection(); -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::establish_http_proxy -// Access: Private -// Description: Establishes a connection to the server through a -// proxy. -//////////////////////////////////////////////////////////////////// -bool HTTPChannel:: -establish_http_proxy() { - _bio = new BioPtr(_proxy); - _source = new BioStreamPtr(new IBioStream(_bio)); - return _bio->connect(); -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::establish_https_proxy -// Access: Private -// Description: Establishes a connection to the secure server through -// a proxy. -//////////////////////////////////////////////////////////////////// -bool HTTPChannel:: -establish_https_proxy() { - // First, ask the proxy to open a connection for us. - _bio = new BioPtr(_proxy); - _source = new BioStreamPtr(new IBioStream(_bio)); - if (!_bio->connect()) { - return false; - } - - ostringstream request; - request - << "CONNECT " << _url.get_server() << ":" << _url.get_port() - << " " << get_http_version_string() << "\r\n"; - if (_http_version >= HTTPClient::HV_11) { - request - << "Host: " << _url.get_server() << "\r\n"; - } - string connect_header = request.str(); - - // Now issue the request and read the response from the proxy. - - string old_proxy_authorization = _client->_proxy_authorization; - bool connected = send_request(connect_header, string(), false); - if (!connected && get_status_code() == 407 && - _client->_proxy_authorization != old_proxy_authorization) { - // If we ended up with a 407 (proxy authorization required), and - // we changed authorization strings recently, then try the new - // authorization string, once. (Normally, send_request() would - // have tried it again automatically, but we may have prevented - // that by passing false allow_reconnect as false.) - if (!prepare_for_next(true)) { - free_bio(); - _bio = new BioPtr(_proxy); - _source = new BioStreamPtr(new IBioStream(_bio)); - if (!_bio->connect()) { - return false; - } - } - connected = send_request(connect_header, string(), false); - } - - if (!connected) { downloader_cat.info() - << "proxy would not open connection to " << _url.get_authority() - << ": " << get_status_code() << " " - << get_status_string() << "\n"; - - if (_client->get_verify_ssl() == HTTPClient::VS_no_verify) { - // If the proxy refused to open a raw connection for us, see - // if it will handle the https communication itself. For - // other error codes, just return error. (We can only - // reliably do this if verify_ssl is minimal, since we're not - // sure whether to trust the proxy to do the verification for - // us.) - if ((get_status_code() / 100) == 4) { - free_bio(); - return establish_http_proxy(); - } - } + << "Could not connect to " << _bio->get_server_name() << ":" + << _bio->get_port() << "\n"; +#ifdef REPORT_SSL_ERRORS + ERR_print_errors_fp(stderr); +#endif + free_bio(); + _state = S_failure; return false; } if (downloader_cat.is_debug()) { - downloader_cat.debug() - << "connection established to " << _url.get_authority() << "\n"; + downloader_cat.info() + << "Connected to " << _bio->get_server_name() << ":" + << _bio->get_port() << "\n"; } - // Reset the state to make it appear like we just opened this - // connection, even though we've already gone through an HTTP - // handshake. - _state = S_new; + if (!_proxy.empty()) { + _state = S_proxy_ready; - // Also reset the HTTP version, because we don't want to limit - // ourselves to whatever version the proxy returned after - // successfully connecting. - _http_version = _client->get_http_version(); - _http_version_string = _client->get_http_version_string(); - - // Ok, we now have a connection to our actual server, so start - // speaking SSL and then ask for the document we really want. - return make_https_connection(); + } else { + if (_url.get_scheme() == "https") { + _state = S_setup_ssl; + } else { + _state = S_ready; + } + } + return false; } //////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::make_https_connection +// Function: HTTPChannel::run_proxy_ready // Access: Private -// Description: Starts speaking SSL over the opened connection. -// Returns true on success, false if the SSL connection -// cannot be established for some reason. +// Description: This state is reached only after first establishing a +// connection to the proxy, if a proxy is in use. +// +// In the normal http mode, this state immediately +// transitions to S_ready, but in some cases (like +// https-over-proxy) we need to send a special message +// directly to the proxy that is separate from the http +// request we will send to the server. //////////////////////////////////////////////////////////////////// bool HTTPChannel:: -make_https_connection() { - BIO *sbio = BIO_new_ssl(_client->_ssl_ctx, true); - BIO_push(sbio, *_bio); +run_proxy_ready() { + // If there's a request to be sent to the proxy, send it now. + if (!_proxy_request_text.empty()) { + if (!http_send(_proxy_request_text)) { + return true; + } + + // All done sending request. + _state = S_proxy_request_sent; - SSL *ssl; - BIO_get_ssl(sbio, &ssl); - nassertr(ssl != (SSL *)NULL, NULL); - SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY); + } else { + _state = S_ready; + } + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::run_proxy_request_sent +// Access: Private +// Description: This state is reached only after we have sent a +// special message to the proxy and we are waiting for +// the proxy's response. It is not used in the normal +// http-over-proxy case, which does not require a +// special message to the proxy. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +run_proxy_request_sent() { + // Wait for the first line to come back from the server. + string line; + if (!http_getline(line)) { + return true; + } + + if (!parse_http_response(line)) { + _state = S_failure; + return false; + } + + _state = S_proxy_reading_header; + _current_field_name = string(); + _current_field_value = string(); + _headers.clear(); + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::run_proxy_reading_header +// Access: Private +// Description: In this state we are reading the header lines from +// the proxy's response to our special message. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +run_proxy_reading_header() { + if (parse_http_header()) { + return true; + } + + if (get_status_code() == 407 && !_proxy.empty()) { + // 407: not authorized to proxy. Try to get the authorization. + string authenticate_request = get_header_value("Proxy-Authenticate"); + string authorization; + if (get_authorization(authorization, authenticate_request, _proxy, true)) { + if (_client->_proxy_authorization != authorization) { + // Change the authorization. + _client->_proxy_authorization = authorization; + make_proxy_request_text(); + + // Roll the state forward to force a new request. + _state = S_begin_body; + return false; + } + } + } + + if (!is_valid()) { + // Proxy wouldn't open connection. + _state = S_failure; + return false; + } + + // Now we have a tunnel opened through the proxy. + _proxy_tunnel = true; + make_request_text(string()); + + if (_url.get_scheme() == "https") { + _state = S_setup_ssl; + } else { + _state = S_ready; + } + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::run_setup_ssl +// Access: Private +// Description: This state begins elevating our existing, unsecure +// connection to a secure, SSL connection. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +run_setup_ssl() { + _sbio = BIO_new_ssl(_client->_ssl_ctx, true); + BIO_push(_sbio, *_bio); if (downloader_cat.is_debug()) { downloader_cat.debug() << "performing SSL handshake\n"; } - if (BIO_do_handshake(sbio) <= 0) { + _state = S_ssl_handshake; + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::run_ssl_handshake +// Access: Private +// Description: This state performs the SSL handshake with the +// server, and also verifies the server's identity when +// the handshake has successfully completed. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +run_ssl_handshake() { + if (BIO_do_handshake(_sbio) <= 0) { + if (BIO_should_retry(_sbio)) { + return true; + } downloader_cat.info() << "Could not establish SSL handshake with " << _url.get_server() << ":" << _url.get_port() << "\n"; @@ -514,19 +531,31 @@ make_https_connection() { #endif // It seems to be an error to free sbio at this point; perhaps // it's already been freed? + _state = S_failure; return false; } + SSL *ssl; + BIO_get_ssl(_sbio, &ssl); + nassertr(ssl != (SSL *)NULL, NULL); + + if (!_nonblocking) { + SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY); + } + // Now that we've made an SSL handshake, we can use the SSL bio to // do all of our communication henceforth. - _bio->set_bio(sbio); + _bio->set_bio(_sbio); + _sbio = NULL; + // Now verify the server is who we expect it to be. long verify_result = SSL_get_verify_result(ssl); if (verify_result == X509_V_ERR_CERT_HAS_EXPIRED) { downloader_cat.info() << "Expired certificate from " << _url.get_server() << ":" << _url.get_port() << "\n"; if (_client->get_verify_ssl() == HTTPClient::VS_normal) { + _state = S_failure; return false; } @@ -535,6 +564,7 @@ make_https_connection() { << "Premature certificate from " << _url.get_server() << ":" << _url.get_port() << "\n"; if (_client->get_verify_ssl() == HTTPClient::VS_normal) { + _state = S_failure; return false; } @@ -543,6 +573,7 @@ make_https_connection() { << "Unable to verify identity of " << _url.get_server() << ":" << _url.get_port() << ", verify error code " << verify_result << "\n"; if (_client->get_verify_ssl() != HTTPClient::VS_no_verify) { + _state = S_failure; return false; } } @@ -553,6 +584,7 @@ make_https_connection() { << "No certificate was presented by server.\n"; if (_client->get_verify_ssl() != HTTPClient::VS_no_verify || !_client->_expected_servers.empty()) { + _state = S_failure; return false; } @@ -579,16 +611,646 @@ make_https_connection() { if (!verify_server(subject)) { downloader_cat.info() << "Server does not match any expected server.\n"; + _state = S_failure; return false; } X509_free(cert); } - return true; + _state = S_ready; + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::run_ready +// Access: Private +// Description: This is the main "ready" state. In this state, we +// have established a (possibly secure) connection to +// the server (or proxy), and the server (or proxy) is +// idle and waiting for us to send a request. +// +// If persistent_connection is true, we will generally +// come back to this state after finishing each request +// on a given connection. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +run_ready() { + // If there's a request to be sent upstream, send it now. + if (!_request_text.empty()) { + if (!http_send(_request_text)) { + return true; + } + } + + // All done sending request. + _state = S_request_sent; + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::run_request_sent +// Access: Private +// Description: In this state we have sent our request to the server +// (or proxy) and we are waiting for a response. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +run_request_sent() { + // Wait for the first line to come back from the server. + string line; + if (!http_getline(line)) { + return true; + } + + if (!parse_http_response(line)) { + _state = S_failure; + return false; + } + + _state = S_reading_header; + _current_field_name = string(); + _current_field_value = string(); + _headers.clear(); + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::run_reading_header +// Access: Private +// Description: In this state we have received the first response to +// our request from the server (or proxy) and we are +// reading the set of header lines preceding the +// requested document. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +run_reading_header() { + if (parse_http_header()) { + return true; + } + + _realm = string(); + + // Look for key properties in the header fields. + _file_size = 0; + string content_length = get_header_value("Content-Length"); + if (!content_length.empty()) { + _file_size = atoi(content_length.c_str()); + } + _redirect = get_header_value("Location"); + + _state = S_read_header; + + // Handle automatic retries and redirects. + int last_status = _last_status_code; + _last_status_code = get_status_code(); + + if (get_status_code() == 407 && last_status != 407 && !_proxy.empty()) { + // 407: not authorized to proxy. Try to get the authorization. + string authenticate_request = get_header_value("Proxy-Authenticate"); + string authorization; + if (get_authorization(authorization, authenticate_request, _proxy, true)) { + if (_client->_proxy_authorization != authorization) { + // Change the authorization. + _client->_proxy_authorization = authorization; + make_request_text(string()); + + // Roll the state forward to force a new request. + _state = S_begin_body; + return false; + } + } + } + + if (get_status_code() == 401 && last_status != 401) { + // 401: not authorized to remote server. Try to get the authorization. + string authenticate_request = get_header_value("WWW-Authenticate"); + string authorization; + if (get_authorization(authorization, authenticate_request, _url, false)) { + make_request_text(authorization); + + // Roll the state forward to force a new request. + _state = S_begin_body; + return false; + } + } + + if ((get_status_code() / 100) == 3 && get_status_code() != 305) { + // Redirect. Should we handle it automatically? + if (!get_redirect().empty() && (_method == "GET" || _method == "HEAD")) { + // Sure! + URLSpec new_url = get_redirect(); + if (!_redirect_trail.insert(new_url).second) { + downloader_cat.warning() + << "cycle detected in redirect to " << new_url << "\n"; + + } else { + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "following redirect to " << new_url << "\n"; + } + if (_url.has_username()) { + new_url.set_username(_url.get_username()); + } + set_url(new_url); + make_header(); + make_request_text(string()); + + // Roll the state forward to force a new request. + _state = S_begin_body; + return false; + } + } + } + + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::run_read_header +// Access: Private +// Description: In this state we have completely read the header +// lines returned by the server (or proxy) in response +// to our request. This state represents the normal +// stopping point of a call to get_document(), etc.; +// further reads will return the body of the request, +// the requested document. +// +// Normally run_read_header() is not called unless the +// user has elected not to read the returned document +// himself. In fact, the state itself only exists so we +// can make a distinction between S_read_header and +// S_begin_body, where S_read_header is safe to return +// to the user and S_begin_body means we need to start +// skipping the document. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +run_read_header() { + _state = S_begin_body; + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::run_begin_body +// Access: Private +// Description: This state begins to skip over the body in +// preparation for making a new request. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +run_begin_body() { + if (!get_persistent_connection() || will_close_connection()) { + // If the socket will close anyway, no point in skipping past the + // previous body; just reset. + free_bio(); + return false; + } + + if (get_status_code() / 100 == 1 || + get_status_code() == 204 || + get_status_code() == 304 || + _method == "HEAD") { + // These status codes, or method HEAD, indicate we have no body. + // Therefore, we have already read the (nonexistent) body. + _state = S_ready; + + } else { + nassertr(_body_stream == NULL, false); + _body_stream = read_body(); + if (_body_stream == (ISocketStream *)NULL) { + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "Unable to skip body.\n"; + } + free_bio(); + + } else { + _state = S_reading_body; + } + } + + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::run_reading_body +// Access: Private +// Description: In this state we are in the process of reading the +// response's body. We will only come to this function +// if the user did not choose to read the entire body +// himself (by calling read_body(), for instance, or +// open_read_file()). +// +// In this case we should skip past the body to reset +// the connection for making a new request. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +run_reading_body() { + if (!get_persistent_connection() || will_close_connection()) { + // If the socket will close anyway, no point in skipping past the + // previous body; just reset. + free_bio(); + return false; + } + + // Skip the body we've already started. + if (_body_stream == NULL) { + // Whoops, we're not in skip-body mode. Better reset. + free_bio(); + return false; + } + + string line; + getline(*_body_stream, line); + while (!_body_stream->fail() && !_body_stream->eof()) { + if (downloader_cat.is_spam()) { + downloader_cat.spam() << "skip: " << line << "\n"; + } + getline(*_body_stream, line); + } + + if (!_body_stream->is_closed()) { + // There's more to come later. + return true; + } + + delete _body_stream; + _body_stream = NULL; + + // This should have been set by the _body_stream finishing. + nassertr(_state != S_reading_body, false); + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::run_read_body +// Access: Private +// Description: In this state we have completely read (or skipped +// over) the body of the response. We should continue +// skipping past the trailer following the body. +// +// Not all bodies come with trailers; in particular, the +// "identity" transfer encoding does not include a +// trailer. It is therefore the responsibility of the +// IdentityStreamBuf or ChunkedStreamBuf to set the +// state appropriately to either S_read_body or +// S_read_trailer following the completion of the body. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +run_read_body() { + if (!get_persistent_connection() || will_close_connection()) { + // If the socket will close anyway, no point in skipping past the + // previous body; just reset. + free_bio(); + return false; + } + // Skip the trailer following the recently-read body. + + string line; + if (!http_getline(line)) { + return true; + } + while (!line.empty()) { + if (!http_getline(line)) { + return true; + } + } + + _state = S_read_trailer; + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::run_read_trailer +// Access: Private +// Description: In this state we have completely read the body and +// the trailer. This state is simply a pass-through +// back to S_ready. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +run_read_trailer() { + if (!get_persistent_connection() || will_close_connection()) { + // If the socket will close anyway, no point in skipping past the + // previous body; just reset. + free_bio(); + return false; + } + + if (!_proxy.empty() && !_proxy_tunnel) { + _state = S_proxy_ready; + } else { + _state = S_ready; + } + return false; } +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::begin_request +// Access: Private +// Description: Begins a new document request to the server, throwing +// away whatever request was currently pending if +// necessary. +//////////////////////////////////////////////////////////////////// +void HTTPChannel:: +begin_request(const string &method, const URLSpec &url, const string &body, + bool nonblocking) { + _status_code = 0; + _status_string = string(); + _redirect_trail.clear(); + _last_status_code = 0; + + // Changing the proxy, or the nonblocking state, is grounds for + // dropping the old connection, if any. + if (_proxy != _client->get_proxy()) { + _proxy = _client->get_proxy(); + free_bio(); + } + + if (_nonblocking != nonblocking) { + _nonblocking = nonblocking; + free_bio(); + } + + _method = method; + set_url(url); + _body = body; + make_header(); + make_request_text(string()); + + if (!_proxy.empty() && _url.get_scheme() == "https") { + // HTTPS over proxy requires tunnelling through the proxy to the + // server so we can handle the SSL connection directly, rather + // than asking the proxy to hand us the particular document(s) in + // question. + ostringstream request; + request + << "CONNECT " << _url.get_server() << ":" << _url.get_port() + << " " << _client->get_http_version_string() << "\r\n"; + if (_client->get_http_version() >= HTTPClient::HV_11) { + request + << "Host: " << _url.get_server() << "\r\n"; + } + _proxy_header = request.str(); + make_proxy_request_text(); + + } else { + _proxy_header = string(); + _proxy_request_text = string(); + } + + // Also, reset from whatever previous request might still be pending. + if (_state == S_failure || (_state < S_read_header && _state != S_ready)) { + free_bio(); + + } else if (_state == S_read_header) { + // Roll one step forwards to start skipping past the previous + // body. + _state = S_begin_body; + } + + _done_state = S_read_header; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::http_getline +// Access: Private +// Description: Reads a single line from the server's reply. Returns +// true if the line is successfully retrieved, or false +// if a complete line has not yet been received or if +// the connection has been closed. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +http_getline(string &str) { + nassertr(!_source.is_null(), false); + int ch = (*_source)->get(); + while (!(*_source)->eof() && !(*_source)->fail()) { + switch (ch) { + case '\n': + // end-of-line character, we're done. + if (downloader_cat.is_spam()) { + downloader_cat.spam() << "recv: " << _working_getline << "\n"; + } + str = _working_getline; + _working_getline = string(); + return true; + + case '\r': + // Ignore CR characters. + break; + + default: + _working_getline += (char)ch; + } + ch = (*_source)->get(); + } + + check_socket(); + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::http_send +// Access: Private +// Description: Sends a series of lines to the server. Returns true +// if the buffer is fully sent, or false if some of it +// remains. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +http_send(const string &str) { + nassertr(str.length() > _sent_so_far, true); + + size_t bytes_to_send = str.length() - _sent_so_far; + int write_count = + BIO_write(*_bio, str.data() + _sent_so_far, bytes_to_send); + + if (write_count <= 0) { + if (BIO_should_retry(*_bio)) { + // Temporary failure: the pipe is full. Wait till later. + return false; + } + // Oops, the connection has been closed! + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "Lost connection to server unexpectedly during write.\n"; + } + free_bio(); + return false; + } + +#ifndef NDEBUG + if (downloader_cat.is_spam()) { + show_send(str.substr(0, write_count)); + } +#endif + + if (write_count < (int)bytes_to_send) { + _sent_so_far += write_count; + return false; + } + + // Buffer completely sent. + _sent_so_far = 0; + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::parse_http_response +// Access: Private +// Description: Parses the first line sent back from an HTTP server +// or proxy and stores the result in _status_code and +// _http_version, etc. Returns true on success, false +// on invalid response. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +parse_http_response(const string &line) { + // The first line back should include the HTTP version and the + // result code. + if (line.length() < 5 || line.substr(0, 5) != "HTTP/") { + // Not an HTTP response. + _status_code = 0; + _status_string = "Not an HTTP response"; + return false; + } + + // Split out the first line into its three components. + size_t p = 5; + while (p < line.length() && !isspace(line[p])) { + p++; + } + _http_version_string = line.substr(0, p); + _http_version = HTTPClient::parse_http_version_string(_http_version_string); + + while (p < line.length() && isspace(line[p])) { + p++; + } + size_t q = p; + while (q < line.length() && !isspace(line[q])) { + q++; + } + string status_code = line.substr(p, q - p); + _status_code = atoi(status_code.c_str()); + + while (q < line.length() && isspace(line[q])) { + q++; + } + _status_string = line.substr(q, line.length() - q); + + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::parse_http_header +// Access: Private +// Description: Reads the series of header lines from the server and +// stores them in _headers. Returns true if there is +// more to read, false when done. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +parse_http_header() { + string line; + if (!http_getline(line)) { + return true; + } + + while (!line.empty()) { + if (isspace(line[0])) { + // If the line begins with a space, that continues the previous + // field. + size_t p = 0; + while (p < line.length() && isspace(line[p])) { + p++; + } + _current_field_value += line.substr(p - 1); + + } else { + // If the line does not begin with a space, that defines a new + // field. + if (!_current_field_name.empty()) { + store_header_field(_current_field_name, _current_field_value); + _current_field_value = string(); + } + + size_t colon = line.find(':'); + if (colon != string::npos) { + _current_field_name = downcase(line.substr(0, colon)); + size_t p = colon + 1; + while (p < line.length() && isspace(line[p])) { + p++; + } + _current_field_value = line.substr(p); + } + } + + if (!http_getline(line)) { + return true; + } + } + + // After reading an empty line, we're done with the headers. + if (!_current_field_name.empty()) { + store_header_field(_current_field_name, _current_field_value); + _current_field_value = string(); + } + + return false; +} + + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::verify_server +// Access: Private +// Description: Returns true if the indicated server matches one of +// our expected servers (or the list of expected servers +// is empty), or false if it does not match any of our +// expected servers. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +verify_server(X509_NAME *subject) const { + if (_client->_expected_servers.empty()) { + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "No expected servers on list; allowing any https connection.\n"; + } + return true; + } + + if (downloader_cat.is_debug()) { + downloader_cat.debug() << "checking server: " << flush; + X509_NAME_print_ex_fp(stderr, subject, 0, 0); + fflush(stderr); + downloader_cat.debug(false) << "\n"; + } + + HTTPClient::ExpectedServers::const_iterator ei; + for (ei = _client->_expected_servers.begin(); + ei != _client->_expected_servers.end(); + ++ei) { + X509_NAME *expected_name = (*ei); + if (x509_name_subset(expected_name, subject)) { + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "Match found!\n"; + } + return true; + } + } + + // None of the lines matched. + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "No match found against any of the following expected servers:\n"; + + for (ei = _client->_expected_servers.begin(); + ei != _client->_expected_servers.end(); + ++ei) { + X509_NAME *expected_name = (*ei); + X509_NAME_print_ex_fp(stderr, expected_name, 0, 0); + fprintf(stderr, "\n"); + } + fflush(stderr); + } + + return false; +} /* Certificate verify error codes: @@ -765,64 +1427,6 @@ certificate signing */ - -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::verify_server -// Access: Private -// Description: Returns true if the indicated server matches one of -// our expected servers (or the list of expected servers -// is empty), or false if it does not match any of our -// expected servers. -//////////////////////////////////////////////////////////////////// -bool HTTPChannel:: -verify_server(X509_NAME *subject) const { - if (_client->_expected_servers.empty()) { - if (downloader_cat.is_debug()) { - downloader_cat.debug() - << "No expected servers on list; allowing any https connection.\n"; - } - return true; - } - - if (downloader_cat.is_debug()) { - downloader_cat.debug() << "checking server: " << flush; - X509_NAME_print_ex_fp(stderr, subject, 0, 0); - fflush(stderr); - downloader_cat.debug(false) << "\n"; - } - - HTTPClient::ExpectedServers::const_iterator ei; - for (ei = _client->_expected_servers.begin(); - ei != _client->_expected_servers.end(); - ++ei) { - X509_NAME *expected_name = (*ei); - if (x509_name_subset(expected_name, subject)) { - if (downloader_cat.is_debug()) { - downloader_cat.debug() - << "Match found!\n"; - } - return true; - } - } - - // None of the lines matched. - if (downloader_cat.is_debug()) { - downloader_cat.debug() - << "No match found against any of the following expected servers:\n"; - - for (ei = _client->_expected_servers.begin(); - ei != _client->_expected_servers.end(); - ++ei) { - X509_NAME *expected_name = (*ei); - X509_NAME_print_ex_fp(stderr, expected_name, 0, 0); - fprintf(stderr, "\n"); - } - fflush(stderr); - } - - return false; -} - //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::get_x509_name_component // Access: Private, Static @@ -880,19 +1484,20 @@ x509_name_subset(X509_NAME *name_a, X509_NAME *name_b) { // Function: HTTPChannel::make_header // Access: Private // Description: Formats the appropriate GET or POST (or whatever) -// request to send to the server. Also saves the -// indicated url. +// request to send to the server, based on the current +// _method, _url, _body, and _proxy settings. //////////////////////////////////////////////////////////////////// void HTTPChannel:: -make_header(string &header, const string &method, - const URLSpec &url, const string &body) { - set_url(url); - _method = method; - +make_header() { string path; - if (_proxy.empty()) { + if (_proxy.empty() || _url.get_scheme() == "https") { + // In either of these cases, we contact the server directly for + // the document, so we just need the server-relative path. path = _url.get_path(); + } else { + // In this case (http-over-proxy), we ask the proxy for the + // document, so we need its full URL. URLSpec url_no_username = _url; url_no_username.set_username(string()); path = url_no_username.get_url(); @@ -901,9 +1506,10 @@ make_header(string &header, const string &method, ostringstream stream; stream - << method << " " << path << " " << get_http_version_string() << "\r\n"; + << _method << " " << path << " " + << _client->get_http_version_string() << "\r\n"; - if (_http_version >= HTTPClient::HV_11) { + if (_client->get_http_version() >= HTTPClient::HV_11) { stream << "Host: " << _url.get_server() << "\r\n"; if (!get_persistent_connection()) { @@ -912,15 +1518,65 @@ make_header(string &header, const string &method, } } - if (!body.empty()) { + if (!_body.empty()) { stream << "Content-Type: application/x-www-form-urlencoded\r\n" - << "Content-Length: " << body.length() << "\r\n"; + << "Content-Length: " << _body.length() << "\r\n"; } - header = stream.str(); + _header = stream.str(); } +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::make_proxy_request_text +// Access: Private +// Description: Builds the _proxy_request_text string. This is a +// special request that will be sent directly to the +// proxy prior to the request tailored for the server. +// Generally this is used to open a tunnelling +// connection for https-over-proxy. +//////////////////////////////////////////////////////////////////// +void HTTPChannel:: +make_proxy_request_text() { + _proxy_request_text = _proxy_header; + + if (!_client->_proxy_authorization.empty()) { + _proxy_request_text += "Proxy-Authorization: "; + _proxy_request_text += _client->_proxy_authorization; + _proxy_request_text += "\r\n"; + } + + _proxy_request_text += "\r\n"; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::make_request_text +// Access: Private +// Description: Builds the _request_text string. This is the +// specific request that will be sent to the server this +// pass, based on the current header and body. +//////////////////////////////////////////////////////////////////// +void HTTPChannel:: +make_request_text(const string &authorization) { + _request_text = _header; + + if (!_proxy.empty() && !_client->_proxy_authorization.empty() && + !_proxy_tunnel) { + _request_text += "Proxy-Authorization: "; + _request_text += _client->_proxy_authorization; + _request_text += "\r\n"; + } + + if (!authorization.empty()) { + _request_text += "Authorization: "; + _request_text += authorization; + _request_text += "\r\n"; + } + + _request_text += "\r\n"; + _request_text += _body; +} + //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::set_url // Access: Private @@ -943,174 +1599,6 @@ set_url(const URLSpec &url) { _url = url; } -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::issue_request -// Access: Private -// Description: Issues the request to the HTTP server and waits for a -// response. -//////////////////////////////////////////////////////////////////// -void HTTPChannel:: -issue_request(const string &header, const string &body) { - if (!_bio.is_null()) { - string request = header; - request += "\r\n"; - request += body; -#ifndef NDEBUG - if (downloader_cat.is_spam()) { - show_send(request); - } -#endif - BIO_puts(*_bio, request.c_str()); - read_http_response(); - - if ((*_source)->eof() || (*_source)->fail()) { - if (downloader_cat.is_debug()) { - downloader_cat.debug() - << "Whoops, socket closed.\n"; - free_bio(); - if (prepare_for_next(true)) { -#ifndef NDEBUG - if (downloader_cat.is_spam()) { - show_send(request); - } -#endif - BIO_puts(*_bio, request.c_str()); - read_http_response(); - } - } - } - } -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::read_http_response -// Access: Private -// Description: Reads all of the responses from the server up until -// the first blank line, and stores the list of header -// key:value pairs so retrieved. -//////////////////////////////////////////////////////////////////// -void HTTPChannel:: -read_http_response() { - _headers.clear(); - _realm = string(); - - // The first line back should include the HTTP version and the - // result code. - string line; - getline(**_source, line); - if (!line.empty() && line[line.length() - 1] == '\r') { - line = line.substr(0, line.length() - 1); - } - if (downloader_cat.is_spam()) { - downloader_cat.spam() << "recv: " << line << "\n"; - } - - if (!(**_source) || line.length() < 5 || line.substr(0, 5) != "HTTP/") { - // Not an HTTP response. - _status_code = 0; - _status_string = "Not an HTTP response"; - return; - } - - // Split out the first line into its three components. - size_t p = 5; - while (p < line.length() && !isspace(line[p])) { - p++; - } - _http_version_string = line.substr(0, p); - _http_version = HTTPClient::parse_http_version_string(_http_version_string); - - while (p < line.length() && isspace(line[p])) { - p++; - } - size_t q = p; - while (q < line.length() && !isspace(line[q])) { - q++; - } - string status_code = line.substr(p, q - p); - _status_code = atoi(status_code.c_str()); - - while (q < line.length() && isspace(line[q])) { - q++; - } - _status_string = line.substr(q, line.length() - q); - - // Now read the rest of the lines. These will be field: value - // pairs. - string field_name; - string field_value; - - getline(**_source, line); - if (!line.empty() && line[line.length() - 1] == '\r') { - line = line.substr(0, line.length() - 1); - } - if (downloader_cat.is_spam()) { - downloader_cat.spam() << "recv: " << line << "\n"; - } - - while (!(*_source)->eof() && !(*_source)->fail() && !line.empty()) { - if (isspace(line[0])) { - // If the line begins with a space, that continues the previous - // field. - p = 0; - while (p < line.length() && isspace(line[p])) { - p++; - } - field_value += line.substr(p - 1); - - } else { - // If the line does not begin with a space, that defines a new - // field. - if (!field_name.empty()) { - store_header_field(field_name, field_value); - field_value = string(); - } - - size_t colon = line.find(':'); - if (colon != string::npos) { - field_name = downcase(line.substr(0, colon)); - p = colon + 1; - while (p < line.length() && isspace(line[p])) { - p++; - } - field_value = line.substr(p); - } - } - - getline(**_source, line); - if (!line.empty() && line[line.length() - 1] == '\r') { - line = line.substr(0, line.length() - 1); - } - if (downloader_cat.is_spam()) { - downloader_cat.spam() << "recv: " << line << "\n"; - } - } - if (!field_name.empty()) { - store_header_field(field_name, field_value); - field_value = string(); - } - - // A blank line terminates the headers. - _state = S_read_header; - - if (get_status_code() / 100 == 1 || - get_status_code() == 204 || - get_status_code() == 304 || - (_method == "HEAD" || _method == "CONNECT")) { - // These status codes, or method HEAD or CONNECT, indicate we have - // no body. Therefore, we have already read the (nonexistent) - // body. - _state = S_read_trailer; - } - - _file_size = 0; - string content_length = get_header_value("Content-Length"); - if (!content_length.empty()) { - _file_size = atoi(content_length.c_str()); - } - _redirect = get_header_value("Location"); -} - //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::store_header_field // Access: Private @@ -1366,101 +1854,6 @@ parse_authentication_schemes(HTTPChannel::AuthenticationSchemes &schemes, } } -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::prepare_for_next -// Access: Private -// Description: Resets the state to prepare it for sending a new -// request to the server. This might mean closing the -// connection and opening a new one, or it might mean -// skipping past the unread body in the persistent -// connection, or it might do nothing at all if the body -// has already been completely read. -// -// If allow_reconnect is true, then the current -// connection may be automatically dropped and a new -// connection reestablished if necessary; otherwise, -// this function will fail (and return false) if -// multiple connections are required. -//////////////////////////////////////////////////////////////////// -bool HTTPChannel:: -prepare_for_next(bool allow_reconnect) { - if (get_persistent_connection() && !will_close_connection() && - _proxy == _client->get_proxy()) { - // See if we can reuse the current connection. - if (_state == S_read_header) { - // We have read the header; now skip past the body. - istream *body = read_body(); - if (body != (istream *)NULL) { - string line; - getline(*body, line); - while (!body->fail() && !body->eof()) { - if (downloader_cat.is_spam()) { - downloader_cat.spam() << "skip: " << line << "\n"; - } - getline(*body, line); - } - delete body; - } - } - - if (_state == S_read_body) { - // We have read the body, but there's a trailer to read. - string line; - getline(**_source, line); - if (!line.empty() && line[line.length() - 1] == '\r') { - line = line.substr(0, line.length() - 1); - } - if (downloader_cat.is_spam()) { - downloader_cat.spam() << "skip: " << line << "\n"; - } - while (!(*_source)->eof() && !(*_source)->fail() && !line.empty()) { - getline(**_source, line); - if (!line.empty() && line[line.length() - 1] == '\r') { - line = line.substr(0, line.length() - 1); - } - if (downloader_cat.is_spam()) { - downloader_cat.spam() << "skip: " << line << "\n"; - } - } - _state = S_read_trailer; - } - - if (_state == S_read_trailer) { - // Great; this connection is ready to go! - return true; - } - } - - if (!_bio.is_null() && _state == S_new) { - // If we have a BIO and the _state is S_new, then we haven't done - // anything with the BIO yet, so we can still use it. - return true; - } - - // Either the client will close the connection after reading the - // body, or we were only partly through reading the body elsewhere; - // or possibly we don't have a connection yet at all. In any case, - // we must now get a new connection. - if (!_bio.is_null() && !allow_reconnect) { - // We have a connection, and we're not allowed to throw it away. - // Too bad. - return false; - } - - // Go ahead and close the old BIO. - free_bio(); - - _proxy = _client->get_proxy(); - _http_version = _client->get_http_version(); - _http_version_string = _client->get_http_version_string(); - if (establish_connection()) { - return true; - } - - free_bio(); - return false; -} - //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::free_bio // Access: Private @@ -1470,8 +1863,15 @@ prepare_for_next(bool allow_reconnect) { //////////////////////////////////////////////////////////////////// void HTTPChannel:: free_bio() { + if (_body_stream != (ISocketStream *)NULL) { + delete _body_stream; + _body_stream = (ISocketStream *)NULL; + } _source.clear(); _bio.clear(); + _working_getline = string(); + _sent_so_far = 0; + _proxy_tunnel = false; _read_index++; _state = S_new; } diff --git a/panda/src/downloader/httpChannel.h b/panda/src/downloader/httpChannel.h index 14976e1235..7a22ab6d5b 100644 --- a/panda/src/downloader/httpChannel.h +++ b/panda/src/downloader/httpChannel.h @@ -35,6 +35,7 @@ #include "bioStreamPtr.h" #include "pmap.h" #include "pointerTo.h" +#include "config_downloader.h" #include class HTTPClient; @@ -58,11 +59,6 @@ class EXPCL_PANDAEXPRESS HTTPChannel : public VirtualFile { private: HTTPChannel(HTTPClient *client); - bool send_request(const string &method, const URLSpec &url, - const string &body); - bool send_request(const string &header, const string &body, - bool allow_reconnect); - public: virtual ~HTTPChannel(); @@ -96,26 +92,48 @@ PUBLISHED: INLINE bool get_document(const URLSpec &url); INLINE bool get_header(const URLSpec &url); + INLINE void request_post_form(const URLSpec &url, const string &body); + INLINE void request_document(const URLSpec &url); + INLINE void request_header(const URLSpec &url); + bool run(); + ISocketStream *read_body(); private: - bool establish_connection(); - bool establish_http(); - bool establish_https(); - bool establish_http_proxy(); - bool establish_https_proxy(); + bool run_connecting(); + bool run_proxy_ready(); + bool run_proxy_request_sent(); + bool run_proxy_reading_header(); + bool run_setup_ssl(); + bool run_ssl_handshake(); + bool run_ready(); + bool run_request_sent(); + bool run_reading_header(); + bool run_read_header(); + bool run_begin_body(); + bool run_reading_body(); + bool run_read_body(); + bool run_read_trailer(); - bool make_https_connection(); + void begin_request(const string &method, const URLSpec &url, + const string &body, bool nonblocking); + + bool http_getline(string &str); + bool http_send(const string &str); + bool parse_http_response(const string &line); + bool parse_http_header(); + + INLINE void check_socket(); bool verify_server(X509_NAME *subject) const; static string get_x509_name_component(X509_NAME *name, int nid); static bool x509_name_subset(X509_NAME *name_a, X509_NAME *name_b); - void make_header(string &header, const string &method, - const URLSpec &url, const string &body); + void make_header(); + void make_proxy_request_text(); + void make_request_text(const string &authorization); + void set_url(const URLSpec &url); - void issue_request(const string &header, const string &body); - void read_http_response(); void store_header_field(const string &field_name, const string &field_value); bool get_authorization(string &authorization, const string &authenticate_request, @@ -129,7 +147,6 @@ private: static void show_send(const string &message); #endif - bool prepare_for_next(bool allow_reconnect); void free_bio(); HTTPClient *_client; @@ -137,18 +154,13 @@ private: PT(BioPtr) _bio; PT(BioStreamPtr) _source; bool _persistent_connection; + bool _nonblocking; URLSpec _url; string _method; + string _header; + string _body; - enum State { - S_new, - S_read_header, - S_started_body, - S_read_body, - S_read_trailer - }; - State _state; int _read_index; HTTPClient::HTTPVersion _http_version; @@ -163,6 +175,45 @@ private: size_t _file_size; + // These members are used to maintain the current state while + // communicating with the server. We need to store everything in + // the class object instead of using local variables because in the + // case of nonblocking I/O we have to be able to return to the + // caller after any I/O operation and resume later where we left + // off. + enum State { + S_new, + S_connecting, + S_proxy_ready, + S_proxy_request_sent, + S_proxy_reading_header, + S_setup_ssl, + S_ssl_handshake, + S_ready, + S_request_sent, + S_reading_header, + S_read_header, + S_begin_body, + S_reading_body, + S_read_body, + S_read_trailer, + S_failure + }; + State _state; + State _done_state; + string _proxy_header; + string _proxy_request_text; + bool _proxy_tunnel; + string _request_text; + string _working_getline; + size_t _sent_so_far; + string _current_field_name; + string _current_field_value; + ISocketStream *_body_stream; + BIO *_sbio; + pset _redirect_trail; + int _last_status_code; + typedef pmap Tokens; typedef pmap AuthenticationSchemes; static void parse_authentication_schemes(AuthenticationSchemes &schemes, diff --git a/panda/src/downloader/identityStreamBuf.cxx b/panda/src/downloader/identityStreamBuf.cxx index 0cde99afe0..ca45ba320f 100644 --- a/panda/src/downloader/identityStreamBuf.cxx +++ b/panda/src/downloader/identityStreamBuf.cxx @@ -111,6 +111,7 @@ underflow() { if (read_count != num_bytes) { // Oops, we didn't read what we thought we would. if (read_count == 0) { + gbump(num_bytes); return EOF; } @@ -133,33 +134,24 @@ underflow() { //////////////////////////////////////////////////////////////////// size_t IdentityStreamBuf:: read_chars(char *start, size_t length) { + size_t read_count = 0; + if (!_has_content_length) { // If we have no restrictions on content length, read till end of // file. (*_source)->read(start, length); - length = (*_source)->gcount(); - - if (length == 0) { - // End of file; we're done. - if (_doc != (HTTPChannel *)NULL && _read_index == _doc->_read_index) { - // An IdentityStreamBuf doesn't have a trailer, so we've already - // "read" it. - _doc->_state = HTTPChannel::S_read_trailer; - } - } + read_count = (*_source)->gcount(); } else { - if (_bytes_remaining == 0) { - return 0; - } - // Extract some of the bytes remaining in the chunk. - - length = min(length, _bytes_remaining); - (*_source)->read(start, length); - length = (*_source)->gcount(); - _bytes_remaining -= length; - + + if (_bytes_remaining != 0) { + length = min(length, _bytes_remaining); + (*_source)->read(start, length); + read_count = (*_source)->gcount(); + _bytes_remaining -= read_count; + } + if (_bytes_remaining == 0) { // We're done. if (_doc != (HTTPChannel *)NULL && _read_index == _doc->_read_index) { @@ -169,8 +161,20 @@ read_chars(char *start, size_t length) { } } } + + if (read_count == 0) { + if ((*_source)->is_closed()) { + // socket closed; we're done. + if (_doc != (HTTPChannel *)NULL && _read_index == _doc->_read_index) { + // An IdentityStreamBuf doesn't have a trailer, so we've already + // "read" it. + _doc->_state = HTTPChannel::S_read_trailer; + } + } + return 0; + } - return length; + return read_count; } #endif // HAVE_SSL