Use hash_table_get instead of hash_table_get_pair.

[wget] / src / retr.c

diff --git a/src/retr.c b/src/retr.c
index 25c5dcf4440e11da851db0e3101d48ead723aa8b..6204839c62c4b331b4a80d0c731ee3fd18134d5e 100644 (file)
--- a/src/retr.c
+++ b/src/retr.c
@@ -139,13 +139,16 @@ limit_bandwidth (wgint bytes, struct ptimer *timer)
 
 /* Write data in BUF to OUT.  However, if *SKIP is non-zero, skip that
    amount of data and decrease SKIP.  Increment *TOTAL by the amount
 
 /* Write data in BUF to OUT.  However, if *SKIP is non-zero, skip that
    amount of data and decrease SKIP.  Increment *TOTAL by the amount

-   of data written.  */
+   of data written.  If OUT2 is not NULL, also write BUF to OUT2.
+   In case of error writing to OUT, -1 is returned.  In case of error
+   writing to OUT2, -2 is returned.  In case of any other error,
+   1 is returned.  */

 
 static int
 
 static int

-write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
-            wgint *written)
+write_data (FILE *out, FILE *out2, const char *buf, int bufsize,
+            wgint *skip, wgint *written)

 {
 {

-  if (!out)
+  if (out == NULL && out2 == NULL)

     return 1;
   if (*skip > bufsize)
     {
     return 1;
   if (*skip > bufsize)
     {


@@ -161,7 +164,10 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
         return 1;
     }
 
         return 1;
     }
 

-  fwrite (buf, 1, bufsize, out);
+  if (out != NULL)
+    fwrite (buf, 1, bufsize, out);
+  if (out2 != NULL)
+    fwrite (buf, 1, bufsize, out2);

   *written += bufsize;
 
   /* Immediately flush the downloaded data.  This should not hinder
   *written += bufsize;
 
   /* Immediately flush the downloaded data.  This should not hinder


@@ -178,9 +184,17 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
      actual justification.  (Also, why 16K?  Anyone test other values?)
   */
 #ifndef __VMS
      actual justification.  (Also, why 16K?  Anyone test other values?)
   */
 #ifndef __VMS

-  fflush (out);
+  if (out != NULL)
+    fflush (out);
+  if (out2 != NULL)
+    fflush (out2);

 #endif /* ndef __VMS */
 #endif /* ndef __VMS */

-  return !ferror (out);
+  if (out != NULL && ferror (out))
+    return -1;
+  else if (out2 != NULL && ferror (out2))
+    return -2;
+  else
+    return 0;

 }
 
 /* Read the contents of file descriptor FD until it the connection
 }
 
 /* Read the contents of file descriptor FD until it the connection


@@ -198,18 +212,26 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
    the amount of data written to disk.  The time it took to download
    the data is stored to ELAPSED.
 
    the amount of data written to disk.  The time it took to download
    the data is stored to ELAPSED.
 

+   If OUT2 is non-NULL, the contents is also written to OUT2.
+   OUT2 will get an exact copy of the response: if this is a chunked
+   response, everything -- including the chunk headers -- is written
+   to OUT2.  (OUT will only get the unchunked response.)
+

    The function exits and returns the amount of data read.  In case of
    error while reading data, -1 is returned.  In case of error while
    The function exits and returns the amount of data read.  In case of
    error while reading data, -1 is returned.  In case of error while

-   writing data, -2 is returned.  */
+   writing data to OUT, -2 is returned.  In case of error while writing
+   data to OUT2, -3 is returned.  */

 
 int
 fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
 
 int
 fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,

-              wgint *qtyread, wgint *qtywritten, double *elapsed, int flags)
+              wgint *qtyread, wgint *qtywritten, double *elapsed, int flags,
+              FILE *out2)

 {
   int ret = 0;
 {
   int ret = 0;

-
-  int dlbufsize = BUFSIZ;
-  char *dlbuf = xmalloc (BUFSIZ);
+#undef max
+#define max(a,b) ((a) > (b) ? (a) : (b))
+  int dlbufsize = max (BUFSIZ, 8 * 1024);
+  char *dlbuf = xmalloc (dlbufsize);

 
   struct ptimer *timer = NULL;
   double last_successful_read_tm = 0;
 
   struct ptimer *timer = NULL;
   double last_successful_read_tm = 0;


@@ -286,13 +308,24 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
                   ret = -1;
                   break;
                 }
                   ret = -1;
                   break;
                 }

+              else if (out2 != NULL)
+                fwrite (line, 1, strlen (line), out2);

 
               remaining_chunk_size = strtol (line, &endl, 16);
 
               remaining_chunk_size = strtol (line, &endl, 16);

+              xfree (line);
+

               if (remaining_chunk_size == 0)
                 {
                   ret = 0;
               if (remaining_chunk_size == 0)
                 {
                   ret = 0;

-                  if (fd_read_line (fd) == NULL)
+                  line = fd_read_line (fd);
+                  if (line == NULL)

                     ret = -1;
                     ret = -1;

+                  else
+                    {
+                      if (out2 != NULL)
+                        fwrite (line, 1, strlen (line), out2);
+                      xfree (line);
+                    }

                   break;
                 }
             }
                   break;
                 }
             }


@@ -342,20 +375,30 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
       if (ret > 0)
         {
           sum_read += ret;
       if (ret > 0)
         {
           sum_read += ret;

-          if (!write_data (out, dlbuf, ret, &skip, &sum_written))
+          int write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written);
+          if (write_res != 0)

             {
             {

-              ret = -2;
+              ret = (write_res == -3) ? -3 : -2;

               goto out;
             }
           if (chunked)
             {
               remaining_chunk_size -= ret;
               if (remaining_chunk_size == 0)
               goto out;
             }
           if (chunked)
             {
               remaining_chunk_size -= ret;
               if (remaining_chunk_size == 0)

-                if (fd_read_line (fd) == NULL)
-                  {
-                    ret = -1;
-                    break;
-                  }
+                {
+                  char *line = fd_read_line (fd);
+                  if (line == NULL)
+                    {
+                      ret = -1;
+                      break;
+                    }
+                  else
+                    {
+                      if (out2 != NULL)
+                        fwrite (line, 1, strlen (line), out2);
+                      xfree (line);
+                    }
+                }

             }
         }
 
             }
         }
 


@@ -577,6 +620,7 @@ retr_rate (wgint bytes, double secs)
 {
   static char res[20];
   static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };
 {
   static char res[20];
   static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };

+  static const char *rate_names_bits[] = {"b/s", "Kb/s", "Mb/s", "Gb/s" };

   int units;
 
   double dlrate = calc_rate (bytes, secs, &units);
   int units;
 
   double dlrate = calc_rate (bytes, secs, &units);


@@ -584,7 +628,7 @@ retr_rate (wgint bytes, double secs)
      e.g. "1022", "247", "12.5", "2.38".  */
   sprintf (res, "%.*f %s",
            dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2,
      e.g. "1022", "247", "12.5", "2.38".  */
   sprintf (res, "%.*f %s",
            dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2,

-           dlrate, rate_names[units]);
+           dlrate, !opt.report_bps ? rate_names[units]: rate_names_bits[units]);

 
   return res;
 }
 
   return res;
 }


@@ -601,6 +645,11 @@ double
 calc_rate (wgint bytes, double secs, int *units)
 {
   double dlrate;
 calc_rate (wgint bytes, double secs, int *units)
 {
   double dlrate;

+  double bibyte = 1000.0;
+ 
+  if (!opt.report_bps)
+    bibyte = 1024.0;
+

 
   assert (secs >= 0);
   assert (bytes >= 0);
 
   assert (secs >= 0);
   assert (bytes >= 0);


@@ -612,16 +661,17 @@ calc_rate (wgint bytes, double secs, int *units)
        0 and the timer's resolution, assume half the resolution.  */
     secs = ptimer_resolution () / 2.0;
 
        0 and the timer's resolution, assume half the resolution.  */
     secs = ptimer_resolution () / 2.0;
 

-  dlrate = bytes / secs;
-  if (dlrate < 1024.0)
+  dlrate = convert_to_bits (bytes) / secs;
+  if (dlrate < bibyte)

     *units = 0;
     *units = 0;

-  else if (dlrate < 1024.0 * 1024.0)
-    *units = 1, dlrate /= 1024.0;
-  else if (dlrate < 1024.0 * 1024.0 * 1024.0)
-    *units = 2, dlrate /= (1024.0 * 1024.0);
+  else if (dlrate < (bibyte * bibyte))
+    *units = 1, dlrate /= bibyte;
+  else if (dlrate < (bibyte * bibyte * bibyte))
+    *units = 2, dlrate /= (bibyte * bibyte);
+

   else
     /* Maybe someone will need this, one day. */
   else
     /* Maybe someone will need this, one day. */

-    *units = 3, dlrate /= (1024.0 * 1024.0 * 1024.0);
+    *units = 3, dlrate /= (bibyte * bibyte * bibyte);

 
   return dlrate;
 }
 
   return dlrate;
 }


@@ -878,14 +928,14 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
     {
       register_download (u->url, local_file);
 
     {
       register_download (u->url, local_file);
 

-      if (redirection_count && 0 != strcmp (origurl, u->url))
+      if (!opt.spider && redirection_count && 0 != strcmp (origurl, u->url))

         register_redirection (origurl, u->url);
 
       if (*dt & TEXTHTML)
         register_redirection (origurl, u->url);
 
       if (*dt & TEXTHTML)

-        register_html (u->url, local_file);
+        register_html (local_file);

 
       if (*dt & TEXTCSS)
 
       if (*dt & TEXTCSS)

-        register_css (u->url, local_file);
+        register_css (local_file);

     }
 
   if (file)
     }
 
   if (file)


@@ -947,8 +997,7 @@ retrieve_from_file (const char *file, bool html, int *count)
     {
       int dt,url_err;
       uerr_t status;
     {
       int dt,url_err;
       uerr_t status;

-      struct url * url_parsed = url_parse(url, &url_err, iri, true);
-
+      struct url *url_parsed = url_parse (url, &url_err, iri, true);

       if (!url_parsed)
         {
           char *error = url_error (url, url_err);
       if (!url_parsed)
         {
           char *error = url_error (url, url_err);


@@ -1006,9 +1055,7 @@ retrieve_from_file (const char *file, bool html, int *count)
           break;
         }
 
           break;
         }
 

-      /* Need to reparse the url, since it didn't have iri information. */
-      if (opt.enable_iri)
-          parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true);
+      parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true);

 
       if ((opt.recursive || opt.page_requisites)
           && (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url)))
 
       if ((opt.recursive || opt.page_requisites)
           && (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url)))