diff --git a/panda/src/downloader/config_downloader.cxx b/panda/src/downloader/config_downloader.cxx index 2a82da09a7..5b795257f1 100644 --- a/panda/src/downloader/config_downloader.cxx +++ b/panda/src/downloader/config_downloader.cxx @@ -120,8 +120,8 @@ config_downloader.GetInt("http-max-connect-count", 10); // connection-specific certificate may also be specified at runtime on // the HTTPClient object, but this will require having a different // HTTPClient object for each differently-certificated connection. -const string http_client_certificate_filename = -config_downloader.GetString("http-client-certificate-filename", ""); +const Filename http_client_certificate_filename = +Filename::expand_from(config_downloader.GetString("http-client-certificate-filename", "")); const string http_client_certificate_passphrase = config_downloader.GetString("http-client-certificate-passphrase", ""); diff --git a/panda/src/downloader/config_downloader.h b/panda/src/downloader/config_downloader.h index b71d924b30..285d57250c 100644 --- a/panda/src/downloader/config_downloader.h +++ b/panda/src/downloader/config_downloader.h @@ -51,7 +51,7 @@ extern const bool http_proxy_tunnel; extern const double http_connect_timeout; extern const double http_timeout; extern const int http_max_connect_count; -extern const string http_client_certificate_filename; +extern const Filename http_client_certificate_filename; extern const string http_client_certificate_passphrase; #endif diff --git a/panda/src/downloader/httpChannel.I b/panda/src/downloader/httpChannel.I index 537275d6f5..456489b956 100644 --- a/panda/src/downloader/httpChannel.I +++ b/panda/src/downloader/httpChannel.I @@ -26,7 +26,7 @@ //////////////////////////////////////////////////////////////////// INLINE bool HTTPChannel:: is_valid() const { - return (_state != S_failure && (_status_code / 100) == 2 && + return (_state != S_failure && (get_status_code() / 100) == 2 && (_server_response_has_no_body || !_source.is_null())); } @@ -115,19 +115,7 @@ get_http_version_string() const { //////////////////////////////////////////////////////////////////// INLINE int HTTPChannel:: get_status_code() const { - return _status_code; -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::get_status_string -// Access: Published -// Description: Returns the string as returned by the server -// describing the status code for humans. This may or -// may not be meaningful. -//////////////////////////////////////////////////////////////////// -INLINE const string &HTTPChannel:: -get_status_string() const { - return _status_string; + return _status_entry._status_code; } //////////////////////////////////////////////////////////////////// diff --git a/panda/src/downloader/httpChannel.cxx b/panda/src/downloader/httpChannel.cxx index c04169c26d..150577604c 100644 --- a/panda/src/downloader/httpChannel.cxx +++ b/panda/src/downloader/httpChannel.cxx @@ -77,8 +77,7 @@ HTTPChannel(HTTPClient *client) : _got_transfer_file_size = false; _bytes_downloaded = 0; _bytes_requested = 0; - _status_code = 0; - _status_string = string(); + _status_entry = StatusEntry(); _response_type = RT_none; _http_version = _client->get_http_version(); _http_version_string = _client->get_http_version_string(); @@ -176,6 +175,72 @@ open_read_file() const { return ((HTTPChannel *)this)->read_body(); } +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::get_status_string +// Access: Published +// Description: Returns the string as returned by the server +// describing the status code for humans. This may or +// may not be meaningful. +//////////////////////////////////////////////////////////////////// +string HTTPChannel:: +get_status_string() const { + switch (_status_entry._status_code) { + case SC_incomplete: + return "Connection in progress"; + + case SC_internal_error: + return "Internal error"; + + case SC_no_connection: + return "No connection"; + + case SC_timeout: + return "Timeout on connection"; + + case SC_lost_connection: + return "Lost connection"; + + case SC_non_http_response: + return "Non-HTTP response"; + + case SC_invalid_http: + return "Could not understand HTTP response"; + + case SC_socks_invalid_version: + return "Unsupported SOCKS version"; + + case SC_socks_no_acceptable_login_method: + return "No acceptable SOCKS login method"; + + case SC_socks_refused: + return "SOCKS proxy refused connection"; + + case SC_socks_no_connection: + return "SOCKS proxy unable to connect"; + + case SC_ssl_internal_failure: + return "SSL internal failure"; + + case SC_ssl_no_handshake: + return "No SSL handshake"; + + case SC_http_error_watermark: + // This shouldn't be triggered. + return "Internal error"; + + case SC_ssl_invalid_server_certificate: + return "SSL invalid server certificate"; + + case SC_ssl_unexpected_server: + return "Unexpected SSL server"; + + case SC_download_write_error: + return "Error writing to disk"; + } + + return _status_entry._status_string; +} + //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::get_header_value // Access: Published @@ -310,6 +375,7 @@ run() { // consecutive lost connections. downloader_cat.warning() << "Too many lost connections, giving up.\n"; + _status_entry._status_code = SC_lost_connection; _state = S_failure; return false; } @@ -329,6 +395,10 @@ run() { downloader_cat.info() << "Reconnecting to " << _bio->get_server_name() << ":" << _bio->get_port() << "\n"; + } else { + downloader_cat.info() + << "Connecting to " << _bio->get_server_name() << ":" + << _bio->get_port() << "\n"; } _state = S_connecting; @@ -669,7 +739,29 @@ reached_done_state() { << ", _done_state = " << _done_state << "\n"; } - if (_state == S_failure || _download_dest == DD_none) { + if (_state == S_failure) { + // We had to give up. Each proxy we tried, in sequence, failed. + // But maybe the last attempt didn't give us the most informative + // response; go back and find the best one. + if (!_status_list.empty()) { + _status_list.push_back(_status_entry); + if (downloader_cat.is_spam()) { + downloader_cat.spam() + << "Reexamining failure responses.\n"; + } + size_t best_i = 0; + for (size_t i = 1; i < _status_list.size(); i++) { + if (more_useful_status_code(_status_list[i]._status_code, + _status_list[best_i]._status_code)) { + best_i = i; + } + } + _status_entry = _status_list[best_i]; + } + + return false; + + } else if (_download_dest == DD_none) { // All done. return false; @@ -702,13 +794,19 @@ reached_done_state() { bool HTTPChannel:: run_try_next_proxy() { if (_proxy_next_index < _proxies.size()) { - // Try the next proxy in sequence. + // Record the previous proxy's status entry, so we can come back + // to it later if we get nonsense from the remaining proxies. + _status_list.push_back(_status_entry); + _status_entry = StatusEntry(); + + // Now try the next proxy in sequence. _proxy = _proxies[_proxy_next_index]; _proxy_auth = (HTTPAuthorization *)NULL; _proxy_next_index++; close_connection(); reconsider_proxy(); _state = S_connecting; + nassertr(_status_list.size() == _proxy_next_index - 1, false); return false; } @@ -725,8 +823,7 @@ run_try_next_proxy() { //////////////////////////////////////////////////////////////////// bool HTTPChannel:: run_connecting() { - _status_code = 0; - _status_string = string(); + _status_entry = StatusEntry(); if (BIO_do_connect(*_bio) <= 0) { if (BIO_should_retry(*_bio)) { @@ -739,6 +836,7 @@ run_connecting() { #ifdef REPORT_OPENSSL_ERRORS ERR_print_errors_fp(stderr); #endif + _status_entry._status_code = SC_no_connection; _state = S_try_next_proxy; return false; } @@ -776,6 +874,8 @@ run_connecting_wait() { if (fd < 0) { downloader_cat.warning() << "nonblocking socket BIO has no file descriptor.\n"; + // This shouldn't be possible. + _status_entry._status_code = SC_internal_error; _state = S_try_next_proxy; return false; } @@ -802,6 +902,8 @@ run_connecting_wait() { if (errcode < 0) { downloader_cat.warning() << "Error in select.\n"; + // This shouldn't be possible. + _status_entry._status_code = SC_internal_error; _state = S_try_next_proxy; return false; } @@ -815,6 +917,7 @@ run_connecting_wait() { downloader_cat.info() << "Timeout connecting to " << _request.get_url().get_server_and_port() << ".\n"; + _status_entry._status_code = SC_timeout; _state = S_try_next_proxy; return false; } @@ -926,7 +1029,7 @@ run_http_proxy_reading_header() { // differentiate them from similar status codes the destination // server might have returned. if (get_status_code() != 407) { - _status_code += 1000; + _status_entry._status_code += 1000; } _state = S_try_next_proxy; @@ -987,6 +1090,7 @@ run_socks_proxy_greet_reply() { // We only speak Socks5. downloader_cat.info() << "Rejecting Socks version " << (int)reply[0] << "\n"; + _status_entry._status_code = SC_socks_invalid_version; _state = S_try_next_proxy; return false; } @@ -994,10 +1098,7 @@ run_socks_proxy_greet_reply() { if (reply[1] == (char)0xff) { downloader_cat.info() << "Socks server does not accept our available login methods.\n"; - // We plug in the phony status code of 407 here, which is the HTTP - // status code that indicates the proxy didn't like our - // authentication. It's a close enough approximation. - _status_code = 407; + _status_entry._status_code = SC_socks_no_acceptable_login_method; _state = S_try_next_proxy; return false; } @@ -1013,6 +1114,7 @@ run_socks_proxy_greet_reply() { downloader_cat.info() << "Socks server accepted unrequested login method " << (int)reply[1] << "\n"; + _status_entry._status_code = SC_socks_no_acceptable_login_method; _state = S_try_next_proxy; return false; } @@ -1077,6 +1179,7 @@ run_socks_proxy_connect_reply() { downloader_cat.info() << "Rejecting Socks version " << (int)reply[0] << "\n"; close_connection(); // connection is now bad. + _status_entry._status_code = SC_socks_invalid_version; _state = S_try_next_proxy; return false; } @@ -1098,6 +1201,19 @@ run_socks_proxy_connect_reply() { o X'09' to X'FF' unassigned */ + switch (reply[1]) { + case 0x03: + case 0x04: + case 0x05: + // These generally mean the same thing: the SOCKS proxy tried, + // but couldn't reach the host. + _status_entry._status_code = SC_socks_no_connection; + break; + + default: + _status_entry._status_code = SC_socks_refused; + } + close_connection(); // connection is now bad. _state = S_try_next_proxy; return false; @@ -1124,6 +1240,7 @@ run_socks_proxy_connect_reply() { default: downloader_cat.info() << "Unsupported SOCKS address type: " << (int)reply[3] << "\n"; + _status_entry._status_code = SC_socks_invalid_version; _state = S_try_next_proxy; return false; } @@ -1195,6 +1312,7 @@ run_setup_ssl() { #ifdef REPORT_OPENSSL_ERRORS ERR_print_errors_fp(stderr); #endif + _status_entry._status_code = SC_ssl_internal_failure; _state = S_failure; return false; } @@ -1260,6 +1378,7 @@ run_ssl_handshake() { #endif // It seems to be an error to free sbio at this point; perhaps // it's already been freed? + _status_entry._status_code = SC_ssl_no_handshake; _state = S_failure; return false; } @@ -1294,6 +1413,7 @@ run_ssl_handshake() { downloader_cat.info() << "Expired certificate from " << _request.get_url().get_server_and_port() << "\n"; if (_client->get_verify_ssl() == HTTPClient::VS_normal) { + _status_entry._status_code = SC_ssl_invalid_server_certificate; _state = S_failure; return false; } @@ -1302,6 +1422,7 @@ run_ssl_handshake() { downloader_cat.info() << "Premature certificate from " << _request.get_url().get_server_and_port() << "\n"; if (_client->get_verify_ssl() == HTTPClient::VS_normal) { + _status_entry._status_code = SC_ssl_invalid_server_certificate; _state = S_failure; return false; } @@ -1311,6 +1432,7 @@ run_ssl_handshake() { << "Unable to verify identity of " << _request.get_url().get_server_and_port() << ", verify error code " << verify_result << "\n"; if (_client->get_verify_ssl() != HTTPClient::VS_no_verify) { + _status_entry._status_code = SC_ssl_invalid_server_certificate; _state = S_failure; return false; } @@ -1320,8 +1442,10 @@ run_ssl_handshake() { if (cert == (X509 *)NULL) { downloader_cat.info() << "No certificate was presented by server.\n"; + // This shouldn't be possible, per the SSL specs. if (_client->get_verify_ssl() != HTTPClient::VS_no_verify || !_client->_expected_servers.empty()) { + _status_entry._status_code = SC_ssl_invalid_server_certificate; _state = S_failure; return false; } @@ -1352,6 +1476,7 @@ run_ssl_handshake() { if (!verify_server(subject)) { downloader_cat.info() << "Server does not match any expected server.\n"; + _status_entry._status_code = SC_ssl_unexpected_server; _state = S_failure; return false; } @@ -1435,6 +1560,7 @@ run_reading_header() { << "Connection lost while reading HTTP response.\n"; if (_response_type == RT_http_hangup) { // This was our second hangup in a row. Give up. + _status_entry._status_code = SC_lost_connection; _state = S_try_next_proxy; } else { @@ -1453,6 +1579,7 @@ run_reading_header() { << _request.get_url().get_server_and_port() << " in run_reading_header (" << elapsed << " seconds elapsed).\n"; + _status_entry._status_code = SC_timeout; _state = S_try_next_proxy; } } @@ -1476,6 +1603,7 @@ run_reading_header() { if (content_range.empty()) { downloader_cat.warning() << "Got 206 response without Content-Range header!\n"; + _status_entry._status_code = SC_invalid_http; _state = S_failure; return false; @@ -1483,6 +1611,7 @@ run_reading_header() { if (!parse_content_range(content_range)) { downloader_cat.warning() << "Couldn't parse Content-Range: " << content_range << "\n"; + _status_entry._status_code = SC_invalid_http; _state = S_failure; return false; } @@ -1507,6 +1636,7 @@ run_reading_header() { // In case we've got a download in effect, reset the download // position to match our starting byte. if (!reset_download_position(_first_byte_delivered)) { + _status_entry._status_code = SC_invalid_http; _state = S_failure; return false; } @@ -1841,6 +1971,7 @@ run_download_to_file() { if (_download_to_file.fail()) { downloader_cat.warning() << "Error writing to " << _download_to_filename << "\n"; + _status_entry._status_code = SC_download_write_error; _state = S_failure; _download_to_file.close(); return false; @@ -2036,7 +2167,11 @@ reconsider_proxy() { void HTTPChannel:: reset_for_new_request() { reset_download_to(); + _last_status_code = 0; + _status_entry = StatusEntry(); + _status_list.clear(); + _response_type = RT_none; _redirect_trail.clear(); _bytes_downloaded = 0; @@ -2191,6 +2326,7 @@ server_getline_failsafe(string &str) { // Huh, the server hung up on us as soon as we tried to connect. if (_response_type == RT_hangup) { // This was our second immediate hangup in a row. Give up. + _status_entry._status_code = SC_lost_connection; _state = S_try_next_proxy; } else { @@ -2209,6 +2345,7 @@ server_getline_failsafe(string &str) { << _request.get_url().get_server_and_port() << " in server_getline_failsafe (" << elapsed << " seconds elapsed).\n"; + _status_entry._status_code = SC_timeout; _state = S_try_next_proxy; } } @@ -2262,6 +2399,7 @@ server_get_failsafe(string &str, size_t num_bytes) { // Huh, the server hung up on us as soon as we tried to connect. if (_response_type == RT_hangup) { // This was our second immediate hangup in a row. Give up. + _status_entry._status_code = SC_lost_connection; _state = S_try_next_proxy; } else { @@ -2280,6 +2418,7 @@ server_get_failsafe(string &str, size_t num_bytes) { << _request.get_url().get_server_and_port() << " in server_get_failsafe (" << elapsed << " seconds elapsed).\n"; + _status_entry._status_code = SC_timeout; _state = S_try_next_proxy; } } @@ -2358,8 +2497,7 @@ parse_http_response(const string &line) { // result code. if (line.length() < 5 || line.substr(0, 5) != string("HTTP/")) { // Not an HTTP response. - _status_code = 0; - _status_string = "Not an HTTP response"; + _status_entry._status_code = SC_non_http_response; if (_response_type == RT_non_http) { // This was our second non-HTTP response in a row. Give up. _state = S_try_next_proxy; @@ -2389,12 +2527,12 @@ parse_http_response(const string &line) { q++; } string status_code = line.substr(p, q - p); - _status_code = atoi(status_code.c_str()); + _status_entry._status_code = atoi(status_code.c_str()); while (q < line.length() && isspace(line[q])) { q++; } - _status_string = line.substr(q, line.length() - q); + _status_entry._status_string = line.substr(q, line.length() - q); return true; } @@ -3141,6 +3279,52 @@ close_connection() { _read_index++; } +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::more_useful_status_code +// Access: Private, Static +// Description: Returns true if status code a is a more useful value +// (that is, it represents a more-nearly successfully +// connection attempt, or contains more information) +// than b, or false otherwise. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +more_useful_status_code(int a, int b) { + if (a >= 100 && b >= 100) { + // Both represent HTTP responses. Responses from a server (< + // 1000) are better than those from a proxy; we take advantage of + // the fact that we have already added 1000 to proxy responses. + // Except for 407, so let's fix that now. + if (a == 407) { + a += 1000; + } + if (b == 407) { + b += 1000; + } + + // Now just check the series. + int series_a = (a / 100); + int series_b = (b / 100); + + // In general, a lower series is a closer success. + return (series_a < series_b); + } + + if (a < 100 && b < 100) { + // Both represent non-HTTP responses. Here a larger number is + // better. + return (a > b); + } + + if (a < 100) { + // a is a non-HTTP response, while b is an HTTP response. HTTP is + // generally, better, unless we exceeded SC_http_error_watermark. + return (a > SC_http_error_watermark); + } + + // Exactly the opposite case as above. + return (b < SC_http_error_watermark); +} + //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::State output operator diff --git a/panda/src/downloader/httpChannel.h b/panda/src/downloader/httpChannel.h index 1d7c68a44f..52947193d3 100644 --- a/panda/src/downloader/httpChannel.h +++ b/panda/src/downloader/httpChannel.h @@ -75,6 +75,33 @@ public: bool will_close_connection() const; PUBLISHED: + // get_status_code() will either return an HTTP-style status code >= + // 100 (e.g. 404), or one of the following values. In general, + // these are ordered from less-successful to more-successful. + enum StatusCode { + SC_incomplete = 0, + SC_internal_error, + SC_no_connection, + SC_timeout, + SC_lost_connection, + SC_non_http_response, + SC_invalid_http, + SC_socks_invalid_version, + SC_socks_no_acceptable_login_method, + SC_socks_refused, + SC_socks_no_connection, + SC_ssl_internal_failure, + SC_ssl_no_handshake, + + // No one returns this code, but StatusCode values higher than + // this are deemed more successful than any generic HTTP response. + SC_http_error_watermark, + + SC_ssl_invalid_server_certificate, + SC_ssl_unexpected_server, + SC_download_write_error, + }; + INLINE bool is_valid() const; INLINE bool is_connection_ready() const; INLINE const URLSpec &get_url() const; @@ -82,7 +109,7 @@ PUBLISHED: INLINE HTTPEnum::HTTPVersion get_http_version() const; INLINE const string &get_http_version_string() const; INLINE int get_status_code() const; - INLINE const string &get_status_string() const; + string get_status_string() const; INLINE const string &get_www_realm() const; INLINE const string &get_proxy_realm() const; INLINE const URLSpec &get_redirect() const; @@ -220,6 +247,8 @@ private: void reset_to_new(); void close_connection(); + static bool more_useful_status_code(int a, int b); + public: // This is declared public solely so we can make an ostream operator // for it. @@ -249,11 +278,18 @@ public: }; private: + class StatusEntry { + public: + int _status_code; + string _status_string; + }; typedef pvector Proxies; + typedef pvector StatusList; HTTPClient *_client; Proxies _proxies; size_t _proxy_next_index; + StatusList _status_list; URLSpec _proxy; PT(BioPtr) _bio; PT(BioStreamPtr) _source; @@ -302,8 +338,7 @@ private: HTTPEnum::HTTPVersion _http_version; string _http_version_string; - int _status_code; - string _status_string; + StatusEntry _status_entry; URLSpec _redirect; string _proxy_realm; diff --git a/panda/src/downloader/httpClient.cxx b/panda/src/downloader/httpClient.cxx index c9839ac318..e51a2cb7a6 100644 --- a/panda/src/downloader/httpClient.cxx +++ b/panda/src/downloader/httpClient.cxx @@ -640,53 +640,55 @@ load_client_certificate() { } } - // Create an in-memory BIO to read the "file" from the memory - // buffer, and call the low-level routines to read the - // keys from the BIO. - BIO *mbio = BIO_new_mem_buf((void *)_client_certificate_pem.data(), - _client_certificate_pem.length()); - - ERR_clear_error(); - _client_certificate_priv = - PEM_read_bio_PrivateKey(mbio, NULL, NULL, - (char *)_client_certificate_passphrase.c_str()); - - // Rewind the "file" to the beginning in order to read the public - // key (which might appear first in the file). - BIO_reset(mbio); - - ERR_clear_error(); - _client_certificate_pub = - PEM_read_bio_X509(mbio, NULL, NULL, NULL); - - BIO_free(mbio); - - - NotifySeverity sev = NS_debug; - string source = "memory"; - if (!_client_certificate_filename.empty()) { - // Only report status to "info" severity if we have read the - // certificate from a file. If it came from an in-memory image, - // a failure will presumably be handled by whoever set the - // image. - sev = NS_info; - source = _client_certificate_filename; - } - - if (_client_certificate_priv != (EVP_PKEY *)NULL && - _client_certificate_pub != (X509 *)NULL) { - downloader_cat.out(sev) - << "Read client certificate from " << source << "\n"; - - } else { - if (_client_certificate_priv == (EVP_PKEY *)NULL) { - downloader_cat.out(sev) - << "Could not read private key from " << source << "\n"; + if (!_client_certificate_pem.empty()) { + // Create an in-memory BIO to read the "file" from the memory + // buffer, and call the low-level routines to read the + // keys from the BIO. + BIO *mbio = BIO_new_mem_buf((void *)_client_certificate_pem.data(), + _client_certificate_pem.length()); + + ERR_clear_error(); + _client_certificate_priv = + PEM_read_bio_PrivateKey(mbio, NULL, NULL, + (char *)_client_certificate_passphrase.c_str()); + + // Rewind the "file" to the beginning in order to read the public + // key (which might appear first in the file). + BIO_reset(mbio); + + ERR_clear_error(); + _client_certificate_pub = + PEM_read_bio_X509(mbio, NULL, NULL, NULL); + + BIO_free(mbio); + + + NotifySeverity sev = NS_debug; + string source = "memory"; + if (!_client_certificate_filename.empty()) { + // Only report status to "info" severity if we have read the + // certificate from a file. If it came from an in-memory image, + // a failure will presumably be handled by whoever set the + // image. + sev = NS_info; + source = _client_certificate_filename; } - if (_client_certificate_pub == (X509 *)NULL) { - downloader_cat.out(sev) - << "Could not read public key from " << source << "\n"; + if (_client_certificate_priv != (EVP_PKEY *)NULL && + _client_certificate_pub != (X509 *)NULL) { + downloader_cat.out(sev) + << "Read client certificate from " << source << "\n"; + + } else { + if (_client_certificate_priv == (EVP_PKEY *)NULL) { + downloader_cat.out(sev) + << "Could not read private key from " << source << "\n"; + } + + if (_client_certificate_pub == (X509 *)NULL) { + downloader_cat.out(sev) + << "Could not read public key from " << source << "\n"; + } } } }