subdocument

panda/src/downloader/httpChannel.I

@@ -182,7 +182,7 @@ get_persistent_connection() const {
 //               the data.
 //
 //               This only has effect on the nonblocking I/O methods
-//               like request_document(), etc.  The blocking methods
+//               like begin_document(), etc.  The blocking methods
 //               like get_document() always use as much CPU and
 //               bandwidth as they can get.
 ////////////////////////////////////////////////////////////////////
@@ -283,7 +283,7 @@ get_file_size() const {
 ////////////////////////////////////////////////////////////////////
 INLINE bool HTTPChannel::
 post_form(const URLSpec &url, const string &body) {
-  begin_request("POST", url, body, false);
+  begin_request("POST", url, body, false, 0, 0);
   run();
   return is_valid();
 }
@@ -296,7 +296,24 @@ post_form(const URLSpec &url, const string &body) {
 ////////////////////////////////////////////////////////////////////
 INLINE bool HTTPChannel::
 get_document(const URLSpec &url) {
-  begin_request("GET", url, string(), false);
+  begin_request("GET", url, string(), false, 0, 0);
+  run();
+  return is_valid();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPChannel::get_subdocument
+//       Access: Published
+//  Description: Retrieves only the specified byte range of the
+//               indicated document.  If last_byte is 0, it stands for
+//               the last byte of the document.  When a subdocument is
+//               requested, get_file_size() and get_bytes_downloaded()
+//               will report the number of bytes of the subdocument,
+//               not of the complete document.
+////////////////////////////////////////////////////////////////////
+INLINE bool HTTPChannel::
+get_subdocument(const URLSpec &url, size_t first_byte, size_t last_byte) {
+  begin_request("GET", url, string(), false, first_byte, last_byte);
   run();
   return is_valid();
 }
@@ -312,17 +329,17 @@ get_document(const URLSpec &url) {
 ////////////////////////////////////////////////////////////////////
 INLINE bool HTTPChannel::
 get_header(const URLSpec &url) {
-  begin_request("HEAD", url, string(), false);
+  begin_request("HEAD", url, string(), false, 0, 0);
   run();
   return is_valid();
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: HTTPChannel::request_post_form
+//     Function: HTTPChannel::begin_post_form
 //       Access: Published
 //  Description: Posts form data to a particular URL and retrieves the
 //               response, all using non-blocking I/O.  See
-//               request_document() and post_form().
+//               begin_document() and post_form().
 //
 //               It is important to note that you *must* call run()
 //               repeatedly after calling this method until run()
@@ -332,12 +349,12 @@ get_header(const URLSpec &url) {
 //               may not get posted.
 ////////////////////////////////////////////////////////////////////
 INLINE void HTTPChannel::
-request_post_form(const URLSpec &url, const string &body) {
+begin_post_form(const URLSpec &url, const string &body) {
-  begin_request("POST", url, body, true);
+  begin_request("POST", url, body, true, 0, 0);
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: HTTPChannel::request_document
+//     Function: HTTPChannel::begin_document
 //       Access: Published
 //  Description: Begins a non-blocking request to retrieve a given
 //               document.  This method will return immediately, even
@@ -351,19 +368,51 @@ request_post_form(const URLSpec &url, const string &body) {
 //               is discarded.
 ////////////////////////////////////////////////////////////////////
 INLINE void HTTPChannel::
-request_document(const URLSpec &url) {
+begin_document(const URLSpec &url) {
-  begin_request("GET", url, string(), true);
+  begin_request("GET", url, string(), true, 0, 0);
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: HTTPChannel::request_header
+//     Function: HTTPChannel::begin_subdocument
 //       Access: Published
-//  Description: Begins a non-blocking request to retrieve a given
+//  Description: Begins a non-blocking request to retrieve only the
-//               header.  See request_document() and get_header().
+//               specified byte range of the indicated document.  If
+//               last_byte is 0, it stands for the last byte of the
+//               document.  When a subdocument is requested,
+//               get_file_size() and get_bytes_downloaded() will
+//               report the number of bytes of the subdocument, not of
+//               the complete document.
 ////////////////////////////////////////////////////////////////////
 INLINE void HTTPChannel::
-request_header(const URLSpec &url) {
+begin_subdocument(const URLSpec &url, size_t first_byte, size_t last_byte) {
-  begin_request("HEAD", url, string(), true);
+  begin_request("GET", url, string(), true, first_byte, last_byte);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPChannel::begin_header
+//       Access: Published
+//  Description: Begins a non-blocking request to retrieve a given
+//               header.  See begin_document() and get_header().
+////////////////////////////////////////////////////////////////////
+INLINE void HTTPChannel::
+begin_header(const URLSpec &url) {
+  begin_request("HEAD", url, string(), true, 0, 0);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPChannel::get_bytes_downloaded
+//       Access: Published
+//  Description: Returns the number of bytes downloaded during the
+//               last (or current) download_to_file() or
+//               download_to_ram operation().  This can be used in
+//               conjunction with get_file_size() to report the
+//               percent complete (but be careful, since
+//               get_file_size() may return 0 if the server has not
+//               told us the size of the file).
+////////////////////////////////////////////////////////////////////
+INLINE size_t HTTPChannel::
+get_bytes_downloaded() const {
+  return _bytes_downloaded;
 }
 
 ////////////////////////////////////////////////////////////////////

panda/src/downloader/httpChannel.cxx

@@ -56,8 +56,11 @@ HTTPChannel(HTTPClient *client) :
   _max_updates_per_second = 1.0f / _seconds_per_update;
   _bytes_per_update = int(_max_bytes_per_second * _seconds_per_update);
   _nonblocking = false;
+  _first_byte = 0;
+  _last_byte = 0;
   _read_index = 0;
   _file_size = 0;
+  _bytes_downloaded = 0;
   _status_code = 0;
   _status_string = string();
   _proxy = _client->get_proxy();
@@ -393,7 +396,7 @@ read_body() {
 //  Description: Specifies the name of a file to download the
 //               resulting document to.  This should be called
 //               immediately after get_document() or
-//               request_document() or related functions.
+//               begin_document() or related functions.
 //
 //               In the case of the blocking I/O methods like
 //               get_document(), this function will download the
@@ -401,7 +404,7 @@ read_body() {
 //               successfully downloaded, false otherwise.
 //
 //               In the case of non-blocking I/O methods like
-//               request_document(), this function simply indicates an
+//               begin_document(), this function simply indicates an
 //               intention to download to the indicated file.  It
 //               returns true if the file can be opened for writing,
 //               false otherwise, but the contents will not be
@@ -440,7 +443,7 @@ download_to_file(const Filename &filename) {
 //  Description: Specifies a Ramfile object to download the
 //               resulting document to.  This should be called
 //               immediately after get_document() or
-//               request_document() or related functions.
+//               begin_document() or related functions.
 //
 //               In the case of the blocking I/O methods like
 //               get_document(), this function will download the
@@ -448,7 +451,7 @@ download_to_file(const Filename &filename) {
 //               was successfully downloaded, false otherwise.
 //
 //               In the case of non-blocking I/O methods like
-//               request_document(), this function simply indicates an
+//               begin_document(), this function simply indicates an
 //               intention to download to the indicated Ramfile.  It
 //               returns true if the file can be opened for writing,
 //               false otherwise, but the contents will not be
@@ -859,10 +862,22 @@ run_reading_header() {
   _realm = string();
 
   // Look for key properties in the header fields.
+  if (get_status_code() == 206) {
+    string content_range = get_header_value("Content-Range");
+    if (!content_range.empty()) {
+      parse_content_range(content_range);
+    }
+  }
+
   _file_size = 0;
   string content_length = get_header_value("Content-Length");
   if (!content_length.empty()) {
     _file_size = atoi(content_length.c_str());
+
+  } else if (get_status_code() == 206 && _last_byte != 0) {
+    // Well, we didn't get a content-length from the server, but we
+    // can infer the number of bytes based on the range we requested.
+    _file_size = _last_byte - _first_byte + 1;
   }
   _redirect = get_header_value("Location");
 
@@ -1126,7 +1141,8 @@ run_download_to_file() {
   int ch = _body_stream->get();
   while (!_body_stream->eof() && !_body_stream->fail()) {
     _download_to_file.put(ch);
-    if (do_throttle && ++count > _bytes_per_update) {
+    _bytes_downloaded++;
+    if (do_throttle && (++count > _bytes_per_update)) {
       // That's enough for now.
       return true;
     }
@@ -1169,7 +1185,8 @@ run_download_to_ram() {
   int ch = _body_stream->get();
   while (!_body_stream->eof() && !_body_stream->fail()) {
     _download_to_ramfile->_data += (char)ch;
-    if (do_throttle && ++count > _bytes_per_update) {
+    _bytes_downloaded++;
+    if (do_throttle && (++count > _bytes_per_update)) {
       // That's enough for now.
       return true;
     }
@@ -1196,12 +1213,14 @@ run_download_to_ram() {
 ////////////////////////////////////////////////////////////////////
 void HTTPChannel::
 begin_request(const string &method, const URLSpec &url, const string &body,
-              bool nonblocking) {
+              bool nonblocking, size_t first_byte, size_t last_byte) {
   reset_download_to();
   _status_code = 0;
   _status_string = string();
   _redirect_trail.clear();
   _last_status_code = 0;
+  _file_size = 0;
+  _bytes_downloaded = 0;
 
   // Changing the proxy, or the nonblocking state, is grounds for
   // dropping the old connection, if any.
@@ -1218,6 +1237,8 @@ begin_request(const string &method, const URLSpec &url, const string &body,
   _method = method;
   set_url(url);
   _body = body;
+  _first_byte = first_byte;
+  _last_byte = last_byte;
   make_header();
   make_request_text(string());
 
@@ -1448,6 +1469,52 @@ parse_http_header() {
   return false;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPChannel::parse_content_range
+//       Access: Private
+//  Description: Interprets the "Content-Range" header in the reply,
+//               and fills in _first_byte and _last_byte appropriately
+//               if the header response can be understood.
+////////////////////////////////////////////////////////////////////
+bool HTTPChannel::
+parse_content_range(const string &content_range) {
+  // First, get the units indication.
+  size_t p = 0;
+  while (p < content_range.length() && !isspace(content_range[p])) {
+    p++;
+  }
+
+  string units = content_range.substr(0, p);
+  while (p < content_range.length() && isspace(content_range[p])) {
+    p++;
+  }
+
+  if (units == "bytes") {
+    const char *c_str = content_range.c_str();
+    char *endptr;
+    if (p < content_range.length() && isdigit(content_range[p])) {
+      long first_byte = strtol(c_str + p, &endptr, 10);
+      p = endptr - c_str;
+      if (p < content_range.length() && content_range[p] == '-') {
+        p++;
+        if (p < content_range.length() && isdigit(content_range[p])) {
+          long last_byte = strtol(c_str + p, &endptr, 10);
+          p = endptr - c_str;
+          
+          if (last_byte >= first_byte) {
+            _first_byte = first_byte;
+            _last_byte = last_byte;
+            return true;
+          }
+        }
+      }
+    }
+  }
+    
+  // Invalid or unhandled response.
+  return false;
+}
+
 
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPChannel::verify_server
@@ -1772,6 +1839,15 @@ make_header() {
     }
   }
 
+  if (_last_byte != 0) {
+    stream 
+      << "Range: bytes=" << _first_byte << "-" << _last_byte << "\r\n";
+
+  } else if (_first_byte != 0) {
+    stream 
+      << "Range: bytes=" << _first_byte << "-\r\n";
+  }
+
   if (!_body.empty()) {
     stream
       << "Content-Type: application/x-www-form-urlencoded\r\n"

panda/src/downloader/httpChannel.h

@@ -101,17 +101,22 @@ PUBLISHED:
 
   INLINE bool post_form(const URLSpec &url, const string &body);
   INLINE bool get_document(const URLSpec &url);
+  INLINE bool get_subdocument(const URLSpec &url, 
+                              size_t first_byte, size_t last_byte);
   INLINE bool get_header(const URLSpec &url);
 
-  INLINE void request_post_form(const URLSpec &url, const string &body);
+  INLINE void begin_post_form(const URLSpec &url, const string &body);
-  INLINE void request_document(const URLSpec &url);
+  INLINE void begin_document(const URLSpec &url);
-  INLINE void request_header(const URLSpec &url);
+  INLINE void begin_subdocument(const URLSpec &url, 
+                                size_t first_byte, size_t last_byte);
+  INLINE void begin_header(const URLSpec &url);
   bool run();
 
   ISocketStream *read_body();
   bool download_to_file(const Filename &filename);
   bool download_to_ram(Ramfile *ramfile);
 
+  INLINE size_t get_bytes_downloaded() const;
   INLINE bool is_download_complete() const;
 
 private:
@@ -135,12 +140,14 @@ private:
   bool run_download_to_ram();
 
   void begin_request(const string &method, const URLSpec &url, 
-                     const string &body, bool nonblocking);
+                     const string &body, bool nonblocking,
+                     size_t first_byte, size_t last_byte);
 
   bool http_getline(string &str);
   bool http_send(const string &str);
   bool parse_http_response(const string &line);
   bool parse_http_header();
+  bool parse_content_range(const string &content_range);
 
   INLINE void check_socket();
   bool verify_server(X509_NAME *subject) const;
@@ -185,6 +192,8 @@ private:
   string _method;
   string _header;
   string _body;
+  size_t _first_byte;
+  size_t _last_byte;
 
   enum DownloadDest {
     DD_none,
@@ -209,6 +218,7 @@ private:
   Headers _headers;
 
   size_t _file_size;
+  size_t _bytes_downloaded;
 
   // These members are used to maintain the current state while
   // communicating with the server.  We need to store everything in