Use hash_table_get instead of hash_table_get_pair.

[wget] / src / retr.c

diff --git a/src/retr.c b/src/retr.c
index 5dc3cc8d660f0692a2ccd52635aa3125d40d4d7d..6204839c62c4b331b4a80d0c731ee3fd18134d5e 100644 (file)
--- a/src/retr.c
+++ b/src/retr.c
@@ -1,6 +1,7 @@
 /* File retrieval.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
 /* File retrieval.
    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,

-   2005, 2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc.
+   2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation,
+   Inc.

 
 This file is part of GNU Wget.
 
 
 This file is part of GNU Wget.
 


@@ -32,9 +33,7 @@ as that of the covered work.  */
 
 #include <stdio.h>
 #include <stdlib.h>
 
 #include <stdio.h>
 #include <stdlib.h>

-#ifdef HAVE_UNISTD_H
-# include <unistd.h>
-#endif /* HAVE_UNISTD_H */
+#include <unistd.h>

 #include <errno.h>
 #include <string.h>
 #include <assert.h>
 #include <errno.h>
 #include <string.h>
 #include <assert.h>


@@ -140,13 +139,16 @@ limit_bandwidth (wgint bytes, struct ptimer *timer)
 
 /* Write data in BUF to OUT.  However, if *SKIP is non-zero, skip that
    amount of data and decrease SKIP.  Increment *TOTAL by the amount
 
 /* Write data in BUF to OUT.  However, if *SKIP is non-zero, skip that
    amount of data and decrease SKIP.  Increment *TOTAL by the amount

-   of data written.  */
+   of data written.  If OUT2 is not NULL, also write BUF to OUT2.
+   In case of error writing to OUT, -1 is returned.  In case of error
+   writing to OUT2, -2 is returned.  In case of any other error,
+   1 is returned.  */

 
 static int
 
 static int

-write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
-            wgint *written)
+write_data (FILE *out, FILE *out2, const char *buf, int bufsize,
+            wgint *skip, wgint *written)

 {
 {

-  if (!out)
+  if (out == NULL && out2 == NULL)

     return 1;
   if (*skip > bufsize)
     {
     return 1;
   if (*skip > bufsize)
     {


@@ -162,7 +164,10 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
         return 1;
     }
 
         return 1;
     }
 

-  fwrite (buf, 1, bufsize, out);
+  if (out != NULL)
+    fwrite (buf, 1, bufsize, out);
+  if (out2 != NULL)
+    fwrite (buf, 1, bufsize, out2);

   *written += bufsize;
 
   /* Immediately flush the downloaded data.  This should not hinder
   *written += bufsize;
 
   /* Immediately flush the downloaded data.  This should not hinder


@@ -179,9 +184,17 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
      actual justification.  (Also, why 16K?  Anyone test other values?)
   */
 #ifndef __VMS
      actual justification.  (Also, why 16K?  Anyone test other values?)
   */
 #ifndef __VMS

-  fflush (out);
+  if (out != NULL)
+    fflush (out);
+  if (out2 != NULL)
+    fflush (out2);

 #endif /* ndef __VMS */
 #endif /* ndef __VMS */

-  return !ferror (out);
+  if (out != NULL && ferror (out))
+    return -1;
+  else if (out2 != NULL && ferror (out2))
+    return -2;
+  else
+    return 0;

 }
 
 /* Read the contents of file descriptor FD until it the connection
 }
 
 /* Read the contents of file descriptor FD until it the connection


@@ -199,18 +212,26 @@ write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
    the amount of data written to disk.  The time it took to download
    the data is stored to ELAPSED.
 
    the amount of data written to disk.  The time it took to download
    the data is stored to ELAPSED.
 

+   If OUT2 is non-NULL, the contents is also written to OUT2.
+   OUT2 will get an exact copy of the response: if this is a chunked
+   response, everything -- including the chunk headers -- is written
+   to OUT2.  (OUT will only get the unchunked response.)
+

    The function exits and returns the amount of data read.  In case of
    error while reading data, -1 is returned.  In case of error while
    The function exits and returns the amount of data read.  In case of
    error while reading data, -1 is returned.  In case of error while

-   writing data, -2 is returned.  */
+   writing data to OUT, -2 is returned.  In case of error while writing
+   data to OUT2, -3 is returned.  */

 
 int
 fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
 
 int
 fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,

-              wgint *qtyread, wgint *qtywritten, double *elapsed, int flags)
+              wgint *qtyread, wgint *qtywritten, double *elapsed, int flags,
+              FILE *out2)

 {
   int ret = 0;
 {
   int ret = 0;

-
-  int dlbufsize = BUFSIZ;
-  char *dlbuf = xmalloc (BUFSIZ);
+#undef max
+#define max(a,b) ((a) > (b) ? (a) : (b))
+  int dlbufsize = max (BUFSIZ, 8 * 1024);
+  char *dlbuf = xmalloc (dlbufsize);

 
   struct ptimer *timer = NULL;
   double last_successful_read_tm = 0;
 
   struct ptimer *timer = NULL;
   double last_successful_read_tm = 0;


@@ -287,13 +308,24 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
                   ret = -1;
                   break;
                 }
                   ret = -1;
                   break;
                 }

+              else if (out2 != NULL)
+                fwrite (line, 1, strlen (line), out2);

 
               remaining_chunk_size = strtol (line, &endl, 16);
 
               remaining_chunk_size = strtol (line, &endl, 16);

+              xfree (line);
+

               if (remaining_chunk_size == 0)
                 {
                   ret = 0;
               if (remaining_chunk_size == 0)
                 {
                   ret = 0;

-                  if (fd_read_line (fd) == NULL)
+                  line = fd_read_line (fd);
+                  if (line == NULL)

                     ret = -1;
                     ret = -1;

+                  else
+                    {
+                      if (out2 != NULL)
+                        fwrite (line, 1, strlen (line), out2);
+                      xfree (line);
+                    }

                   break;
                 }
             }
                   break;
                 }
             }


@@ -343,20 +375,30 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
       if (ret > 0)
         {
           sum_read += ret;
       if (ret > 0)
         {
           sum_read += ret;

-          if (!write_data (out, dlbuf, ret, &skip, &sum_written))
+          int write_res = write_data (out, out2, dlbuf, ret, &skip, &sum_written);
+          if (write_res != 0)

             {
             {

-              ret = -2;
+              ret = (write_res == -3) ? -3 : -2;

               goto out;
             }
           if (chunked)
             {
               remaining_chunk_size -= ret;
               if (remaining_chunk_size == 0)
               goto out;
             }
           if (chunked)
             {
               remaining_chunk_size -= ret;
               if (remaining_chunk_size == 0)

-                if (fd_read_line (fd) == NULL)
-                  {
-                    ret = -1;
-                    break;
-                  }
+                {
+                  char *line = fd_read_line (fd);
+                  if (line == NULL)
+                    {
+                      ret = -1;
+                      break;
+                    }
+                  else
+                    {
+                      if (out2 != NULL)
+                        fwrite (line, 1, strlen (line), out2);
+                      xfree (line);
+                    }
+                }

             }
         }
 
             }
         }
 


@@ -578,6 +620,7 @@ retr_rate (wgint bytes, double secs)
 {
   static char res[20];
   static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };
 {
   static char res[20];
   static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };

+  static const char *rate_names_bits[] = {"b/s", "Kb/s", "Mb/s", "Gb/s" };

   int units;
 
   double dlrate = calc_rate (bytes, secs, &units);
   int units;
 
   double dlrate = calc_rate (bytes, secs, &units);


@@ -585,7 +628,7 @@ retr_rate (wgint bytes, double secs)
      e.g. "1022", "247", "12.5", "2.38".  */
   sprintf (res, "%.*f %s",
            dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2,
      e.g. "1022", "247", "12.5", "2.38".  */
   sprintf (res, "%.*f %s",
            dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2,

-           dlrate, rate_names[units]);
+           dlrate, !opt.report_bps ? rate_names[units]: rate_names_bits[units]);

 
   return res;
 }
 
   return res;
 }


@@ -602,6 +645,11 @@ double
 calc_rate (wgint bytes, double secs, int *units)
 {
   double dlrate;
 calc_rate (wgint bytes, double secs, int *units)
 {
   double dlrate;

+  double bibyte = 1000.0;
+ 
+  if (!opt.report_bps)
+    bibyte = 1024.0;
+

 
   assert (secs >= 0);
   assert (bytes >= 0);
 
   assert (secs >= 0);
   assert (bytes >= 0);


@@ -613,16 +661,17 @@ calc_rate (wgint bytes, double secs, int *units)
        0 and the timer's resolution, assume half the resolution.  */
     secs = ptimer_resolution () / 2.0;
 
        0 and the timer's resolution, assume half the resolution.  */
     secs = ptimer_resolution () / 2.0;
 

-  dlrate = bytes / secs;
-  if (dlrate < 1024.0)
+  dlrate = convert_to_bits (bytes) / secs;
+  if (dlrate < bibyte)

     *units = 0;
     *units = 0;

-  else if (dlrate < 1024.0 * 1024.0)
-    *units = 1, dlrate /= 1024.0;
-  else if (dlrate < 1024.0 * 1024.0 * 1024.0)
-    *units = 2, dlrate /= (1024.0 * 1024.0);
+  else if (dlrate < (bibyte * bibyte))
+    *units = 1, dlrate /= bibyte;
+  else if (dlrate < (bibyte * bibyte * bibyte))
+    *units = 2, dlrate /= (bibyte * bibyte);
+

   else
     /* Maybe someone will need this, one day. */
   else
     /* Maybe someone will need this, one day. */

-    *units = 3, dlrate /= (1024.0 * 1024.0 * 1024.0);
+    *units = 3, dlrate /= (bibyte * bibyte * bibyte);

 
   return dlrate;
 }
 
   return dlrate;
 }


@@ -765,7 +814,7 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
       proxy_url = NULL;
     }
 
       proxy_url = NULL;
     }
 

-  location_changed = (result == NEWLOCATION);
+  location_changed = (result == NEWLOCATION || result == NEWLOCATION_KEEP_POST);

   if (location_changed)
     {
       char *construced_newloc;
   if (location_changed)
     {
       char *construced_newloc;


@@ -839,12 +888,17 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
         }
       u = newloc_parsed;
 
         }
       u = newloc_parsed;
 

-      /* If we're being redirected from POST, we don't want to POST
+      /* If we're being redirected from POST, and we received a
+         redirect code different than 307, we don't want to POST

          again.  Many requests answer POST with a redirection to an
          index page; that redirection is clearly a GET.  We "suspend"
          POST data for the duration of the redirections, and restore
          again.  Many requests answer POST with a redirection to an
          index page; that redirection is clearly a GET.  We "suspend"
          POST data for the duration of the redirections, and restore

-         it when we're done. */
-      if (!post_data_suspended)
+         it when we're done.
+        
+        RFC2616 HTTP/1.1 introduces code 307 Temporary Redirect
+        specifically to preserve the method of the request.
+        */
+      if (result != NEWLOCATION_KEEP_POST && !post_data_suspended)

         SUSPEND_POST_DATA;
 
       goto redirected;
         SUSPEND_POST_DATA;
 
       goto redirected;


@@ -874,14 +928,14 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file,
     {
       register_download (u->url, local_file);
 
     {
       register_download (u->url, local_file);
 

-      if (redirection_count && 0 != strcmp (origurl, u->url))
+      if (!opt.spider && redirection_count && 0 != strcmp (origurl, u->url))

         register_redirection (origurl, u->url);
 
       if (*dt & TEXTHTML)
         register_redirection (origurl, u->url);
 
       if (*dt & TEXTHTML)

-        register_html (u->url, local_file);
+        register_html (local_file);

 
       if (*dt & TEXTCSS)
 
       if (*dt & TEXTCSS)

-        register_css (u->url, local_file);
+        register_css (local_file);

     }
 
   if (file)
     }
 
   if (file)


@@ -943,8 +997,7 @@ retrieve_from_file (const char *file, bool html, int *count)
     {
       int dt,url_err;
       uerr_t status;
     {
       int dt,url_err;
       uerr_t status;

-      struct url * url_parsed = url_parse(url, &url_err, iri, true);
-
+      struct url *url_parsed = url_parse (url, &url_err, iri, true);

       if (!url_parsed)
         {
           char *error = url_error (url, url_err);
       if (!url_parsed)
         {
           char *error = url_error (url, url_err);


@@ -1002,9 +1055,7 @@ retrieve_from_file (const char *file, bool html, int *count)
           break;
         }
 
           break;
         }
 

-      /* Need to reparse the url, since it didn't have iri information. */
-      if (opt.enable_iri)
-          parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true);
+      parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true);

 
       if ((opt.recursive || opt.page_requisites)
           && (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url)))
 
       if ((opt.recursive || opt.page_requisites)
           && (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url)))