[svn] Commit my url.c fix (space as unsafe character) and Jan's

[wget] / src / retr.c

diff --git a/src/retr.c b/src/retr.c
index 3d5a32101af4e9a4f8e6b18de35cc231189ea9a1..800ee5d331aea3761fc086ee83427e829608d8fb 100644 (file)
--- a/src/retr.c
+++ b/src/retr.c
@@ -1,5 +1,5 @@
 /* File retrieval.
-   Copyright (C) 1995, 1996, 1997, 1998 Free Software Foundation, Inc.
+   Copyright (C) 1995, 1996, 1997, 1998, 2000 Free Software Foundation, Inc.
 
 This file is part of Wget.
 
@@ -42,6 +42,7 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
 #include "ftp.h"
 #include "host.h"
 #include "connect.h"
+#include "hash.h"
 
 #ifdef WINDOWS
 LARGE_INTEGER internal_time;
@@ -60,6 +61,8 @@ enum spflags { SP_NONE, SP_INIT, SP_FINISH };
 
 static int show_progress PARAMS ((long, long, enum spflags));
 
+#define MIN(i, j) ((i) <= (j) ? (i) : (j))
+
 /* Reads the contents of file descriptor FD, until it is closed, or a
    read error occurs.  The data is read in 8K chunks, and stored to
    stream fp, which should have been open for writing.  If BUF is
@@ -83,9 +86,9 @@ static int show_progress PARAMS ((long, long, enum spflags));
    from fd immediately, flush or discard the buffer.  */
 int
 get_contents (int fd, FILE *fp, long *len, long restval, long expected,
-             struct rbuf *rbuf)
+             struct rbuf *rbuf, int use_expected)
 {
-  int res;
+  int res = 0;
   static char c[8192];
 
   *len = restval;
@@ -105,10 +108,17 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
          *len += res;
        }
     }
-  /* Read from fd while there is available data.  */
-  do
+  /* Read from fd while there is available data.
+
+     Normally, if expected is 0, it means that it is not known how
+     much data is expected.  However, if use_expected is specified,
+     then expected being zero means exactly that.  */
+  while (!use_expected || (*len < expected))
     {
-      res = iread (fd, c, sizeof (c));
+      int amount_to_read = (use_expected
+                           ? MIN (expected - *len, sizeof (c))
+                           : sizeof (c));
+      res = iread (fd, c, amount_to_read);
       if (res > 0)
        {
          if (fwrite (c, sizeof (char), res, fp) < res)
@@ -120,7 +130,9 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
            }
          *len += res;
        }
-    } while (res > 0);
+      else
+       break;
+    }
   if (res < -1)
     res = -1;
   if (opt.verbose)
@@ -323,7 +335,7 @@ retrieve_url (const char *origurl, char **file, char **newloc,
   int local_use_proxy;
   char *mynewloc, *proxy;
   struct urlinfo *u;
-  slist *redirections;
+  struct hash_table *redirections = NULL;
 
   /* If dt is NULL, just ignore it.  */
   if (!dt)
@@ -334,8 +346,6 @@ retrieve_url (const char *origurl, char **file, char **newloc,
   if (file)
     *file = NULL;
 
-  redirections = NULL;
-
   u = newurl ();
   /* Parse the URL. */
   result = parseurl (url, u, 0);
@@ -343,8 +353,9 @@ retrieve_url (const char *origurl, char **file, char **newloc,
     {
       logprintf (LOG_NOTQUIET, "%s: %s.\n", url, uerrmsg (result));
       freeurl (u, 1);
-      free_slist (redirections);
-      free (url);
+      if (redirections)
+       string_set_free (redirections);
+      xfree (url);
       return result;
     }
 
@@ -379,8 +390,9 @@ retrieve_url (const char *origurl, char **file, char **newloc,
        {
          logputs (LOG_NOTQUIET, _("Could not find proxy host.\n"));
          freeurl (u, 1);
-         free_slist (redirections);
-         free (url);
+         if (redirections)
+           string_set_free (redirections);
+         xfree (url);
          return PROXERR;
        }
       /* Parse the proxy URL.  */
@@ -392,8 +404,9 @@ retrieve_url (const char *origurl, char **file, char **newloc,
          else
            logprintf (LOG_NOTQUIET, _("Proxy %s: Must be HTTP.\n"), proxy);
          freeurl (u, 1);
-         free_slist (redirections);
-         free (url);
+         if (redirections)
+           string_set_free (redirections);
+         xfree (url);
          return PROXERR;
        }
       u->proto = URLHTTP;
@@ -443,7 +456,7 @@ retrieve_url (const char *origurl, char **file, char **newloc,
         there break the rules and use relative URLs, and popular
         browsers are lenient about this, so wget should be too. */
       construced_newloc = url_concat (url, mynewloc);
-      free (mynewloc);
+      xfree (mynewloc);
       mynewloc = construced_newloc;
 
       /* Now, see if this new location makes sense. */
@@ -454,48 +467,44 @@ retrieve_url (const char *origurl, char **file, char **newloc,
          logprintf (LOG_NOTQUIET, "%s: %s.\n", mynewloc, uerrmsg (newloc_result));
          freeurl (newloc_struct, 1);
          freeurl (u, 1);
-         free_slist (redirections);
-         free (url);
-         free (mynewloc);
+         if (redirections)
+           string_set_free (redirections);
+         xfree (url);
+         xfree (mynewloc);
          return result;
        }
 
       /* Now mynewloc will become newloc_struct->url, because if the
          Location contained relative paths like .././something, we
          don't want that propagating as url.  */
-      free (mynewloc);
+      xfree (mynewloc);
       mynewloc = xstrdup (newloc_struct->url);
 
-      /* Check for redirection to back to itself.  */
-      if (!strcmp (u->url, newloc_struct->url))
+      if (!redirections)
        {
-         logprintf (LOG_NOTQUIET, _("%s: Redirection to itself.\n"),
-                    mynewloc);
-         freeurl (newloc_struct, 1);
-         freeurl (u, 1);
-         free_slist (redirections);
-         free (url);
-         free (mynewloc);
-         return WRONGCODE;
+         redirections = make_string_hash_table (0);
+         /* Add current URL immediately so we can detect it as soon
+             as possible in case of a cycle. */
+         string_set_add (redirections, u->url);
        }
 
       /* The new location is OK.  Let's check for redirection cycle by
          peeking through the history of redirections. */
-      if (in_slist (redirections, newloc_struct->url))
+      if (string_set_exists (redirections, newloc_struct->url))
        {
          logprintf (LOG_NOTQUIET, _("%s: Redirection cycle detected.\n"),
                     mynewloc);
          freeurl (newloc_struct, 1);
          freeurl (u, 1);
-         free_slist (redirections);
-         free (url);
-         free (mynewloc);
+         if (redirections)
+           string_set_free (redirections);
+         xfree (url);
+         xfree (mynewloc);
          return WRONGCODE;
        }
+      string_set_add (redirections, newloc_struct->url);
 
-      redirections = add_slist (redirections, newloc_struct->url, NOSORT);
-
-      free (url);
+      xfree (url);
       url = mynewloc;
       freeurl (u, 1);
       u = newloc_struct;
@@ -510,12 +519,13 @@ retrieve_url (const char *origurl, char **file, char **newloc,
        *file = NULL;
     }
   freeurl (u, 1);
-  free_slist (redirections);
+  if (redirections)
+    string_set_free (redirections);
 
   if (newloc)
     *newloc = url;
   else
-    free (url);
+    xfree (url);
 
   return result;
 }
@@ -531,9 +541,7 @@ retrieve_from_file (const char *file, int html, int *count)
   uerr_t status;
   urlpos *url_list, *cur_url;
 
-  /* If spider-mode is on, we do not want get_urls_html barfing
-     errors on baseless links.  */
-  url_list = (html ? get_urls_html (file, NULL, opt.spider, FALSE)
+  url_list = (html ? get_urls_html (file, NULL, FALSE, NULL)
              : get_urls_file (file));
   status = RETROK;             /* Suppose everything is OK.  */
   *count = 0;                  /* Reset the URL count.  */