X-Git-Url: http://sjero.net/git/?a=blobdiff_plain;f=src%2Fretr.c;h=e619f1cbb74b9ef5b1d50131d8d4ad957c5a6684;hb=9228f0bf53d3b42459daeb28372196a007de3014;hp=3d5a32101af4e9a4f8e6b18de35cc231189ea9a1;hpb=986c4450293b7f6203db8c999eafaf8e9d9e372e;p=wget

diff --git a/src/retr.c b/src/retr.c
index 3d5a3210..e619f1cb 100644
--- a/src/retr.c
+++ b/src/retr.c
@@ -1,21 +1,31 @@
 /* File retrieval.
-   Copyright (C) 1995, 1996, 1997, 1998 Free Software Foundation, Inc.
+   Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001 Free Software Foundation, Inc.
 
-This file is part of Wget.
+This file is part of GNU Wget.
 
-This program is free software; you can redistribute it and/or modify
+GNU Wget is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
+the Free Software Foundation; either version 2 of the License, or (at
+your option) any later version.
 
-This program is distributed in the hope that it will be useful,
+GNU Wget is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+along with Wget; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+In addition, as a special exception, the Free Software Foundation
+gives permission to link the code of its release of Wget with the
+OpenSSL project's "OpenSSL" library (or with modified versions of it
+that use the same license as the "OpenSSL" library), and distribute
+the linked executables.  You must obey the GNU General Public License
+in all respects for all of the code used other than "OpenSSL".  If you
+modify this file, you may extend this exception to your version of the
+file, but you are not obligated to do so.  If you do not wish to do
+so, delete this exception statement from your version.  */
 
 #include <config.h>
 
@@ -31,34 +41,76 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
 #else
 # include <strings.h>
 #endif /* HAVE_STRING_H */
-#include <ctype.h>
 #include <assert.h>
 
 #include "wget.h"
 #include "utils.h"
 #include "retr.h"
+#include "progress.h"
 #include "url.h"
 #include "recur.h"
 #include "ftp.h"
 #include "host.h"
 #include "connect.h"
+#include "hash.h"
 
-#ifdef WINDOWS
-LARGE_INTEGER internal_time;
-#else
-/* Internal variables used by the timer.  */
-static long internal_secs, internal_msecs;
+#ifdef HAVE_SSL
+# include "gen_sslfunc.h"	/* for ssl_iread */
+#endif
+
+#ifndef errno
+extern int errno;
 #endif
 
-void logflush PARAMS ((void));
+/* See the comment in gethttp() why this is needed. */
+int global_download_count;
 
-/* From http.c.  */
-uerr_t http_loop PARAMS ((struct urlinfo *, char **, int *));
 
-/* Flags for show_progress().  */
-enum spflags { SP_NONE, SP_INIT, SP_FINISH };
+static struct {
+  long bytes;
+  double dltime;
+} limit_data;
+
+static void
+limit_bandwidth_reset (void)
+{
+  limit_data.bytes  = 0;
+  limit_data.dltime = 0;
+}
+
+/* Limit the bandwidth by pausing the download for an amount of time.
+   BYTES is the number of bytes received from the network, DELTA is
+   how long it took to receive them, DLTIME the current download time,
+   TIMER the timer, and ADJUSTMENT the previous.  */
+
+static void
+limit_bandwidth (long bytes, double delta)
+{
+  double expected;
+
+  limit_data.bytes += bytes;
+  limit_data.dltime += delta;
 
-static int show_progress PARAMS ((long, long, enum spflags));
+  expected = 1000.0 * limit_data.bytes / opt.limit_rate;
+
+  if (expected > limit_data.dltime)
+    {
+      double slp = expected - limit_data.dltime;
+      if (slp < 200)
+	{
+	  DEBUGP (("deferring a %.2f ms sleep (%ld/%.2f).\n",
+		   slp, limit_data.bytes, limit_data.dltime));
+	  return;
+	}
+      DEBUGP (("sleeping %.2f ms\n", slp));
+      usleep ((unsigned long) (1000 * slp));
+    }
+
+  limit_data.bytes = 0;
+  limit_data.dltime = 0;
+}
+
+#define MIN(i, j) ((i) <= (j) ? (i) : (j))
 
 /* Reads the contents of file descriptor FD, until it is closed, or a
    read error occurs.  The data is read in 8K chunks, and stored to
@@ -83,236 +135,185 @@ static int show_progress PARAMS ((long, long, enum spflags));
    from fd immediately, flush or discard the buffer.  */
 int
 get_contents (int fd, FILE *fp, long *len, long restval, long expected,
-	      struct rbuf *rbuf)
+	      struct rbuf *rbuf, int use_expected, double *elapsed)
 {
-  int res;
-  static char c[8192];
+  int res = 0;
+  static char c[16384];
+  void *progress = NULL;
+  struct wget_timer *timer = wtimer_allocate ();
+  double dltime = 0, last_dltime = 0;
 
   *len = restval;
+
   if (opt.verbose)
-    show_progress (restval, expected, SP_INIT);
+    progress = progress_create (restval, expected);
+
   if (rbuf && RBUF_FD (rbuf) == fd)
     {
+      int sz = 0;
       while ((res = rbuf_flush (rbuf, c, sizeof (c))) != 0)
 	{
-	  if (fwrite (c, sizeof (char), res, fp) < res)
-	    return -2;
-	  if (opt.verbose)
-	    {
-	      if (show_progress (res, expected, SP_NONE))
-		fflush (fp);
-	    }
+	  fwrite (c, sizeof (char), res, fp);
 	  *len += res;
+	  sz += res;
 	}
-    }
-  /* Read from fd while there is available data.  */
-  do
-    {
-      res = iread (fd, c, sizeof (c));
-      if (res > 0)
+      if (sz)
+	fflush (fp);
+      if (ferror (fp))
 	{
-	  if (fwrite (c, sizeof (char), res, fp) < res)
-	    return -2;
-	  if (opt.verbose)
-	    {
-	      if (show_progress (res, expected, SP_NONE))
-		fflush (fp);
-	    }
-	  *len += res;
+	  res = -2;
+	  goto out;
 	}
-    } while (res > 0);
-  if (res < -1)
-    res = -1;
-  if (opt.verbose)
-    show_progress (0, expected, SP_FINISH);
-  return res;
-}
-
-static void
-print_percentage (long bytes, long expected)
-{
-  int percentage = (int)(100.0 * bytes / expected);
-  logprintf (LOG_VERBOSE, " [%3d%%]", percentage);
-}
-
-/* Show the dotted progress report of file loading.  Called with
-   length and a flag to tell it whether to reset or not.  It keeps the
-   offset information in static local variables.
-
-   Return value: 1 or 0, designating whether any dots have been drawn.
+      if (progress)
+	progress_update (progress, sz, 0);
+    }
 
-   If the init argument is set, the routine will initialize.
+  if (opt.limit_rate)
+    limit_bandwidth_reset ();
+  wtimer_reset (timer);
 
-   If the res is non-zero, res/line_bytes lines are skipped
-   (meaning the appropriate number ok kilobytes), and the number of
-   "dots" fitting on the first line are drawn as ','.  */
-static int
-show_progress (long res, long expected, enum spflags flags)
-{
-  static long line_bytes;
-  static long offs;
-  static int ndot, nrow;
-  int any_output = 0;
+  /* Read from fd while there is available data.
 
-  if (flags == SP_FINISH)
+     Normally, if expected is 0, it means that it is not known how
+     much data is expected.  However, if use_expected is specified,
+     then expected being zero means exactly that.  */
+  while (!use_expected || (*len < expected))
     {
-      if (expected)
+      int amount_to_read = (use_expected
+			    ? MIN (expected - *len, sizeof (c))
+			    : sizeof (c));
+#ifdef HAVE_SSL
+      if (rbuf->ssl!=NULL)
+	res = ssl_iread (rbuf->ssl, c, amount_to_read);
+      else
+#endif /* HAVE_SSL */
+	res = iread (fd, c, amount_to_read);
+
+      if (res > 0)
 	{
-	  int dot = ndot;
-	  char *tmpstr = (char *)alloca (2 * opt.dots_in_line + 1);
-	  char *tmpp = tmpstr;
-	  for (; dot < opt.dots_in_line; dot++)
+	  fwrite (c, sizeof (char), res, fp);
+	  /* Always flush the contents of the network packet.  This
+	     should not be adverse to performance, as the network
+	     packets typically won't be too tiny anyway.  */
+	  fflush (fp);
+	  if (ferror (fp))
 	    {
-	      if (!(dot % opt.dot_spacing))
-		*tmpp++ = ' ';
-	      *tmpp++ = ' ';
+	      res = -2;
+	      goto out;
 	    }
-	  *tmpp = '\0';
-	  logputs (LOG_VERBOSE, tmpstr);
-	  print_percentage (nrow * line_bytes + ndot * opt.dot_bytes + offs,
-			    expected);
-	}
-      logputs (LOG_VERBOSE, "\n\n");
-      return 0;
-    }
 
-  /* Temporarily disable flushing.  */
-  opt.no_flush = 1;
-  /* init set means initialization.  If res is set, it also means that
-     the retrieval is *not* done from the beginning.  The part that
-     was already retrieved is not shown again.  */
-  if (flags == SP_INIT)
-    {
-      /* Generic initialization of static variables.  */
-      offs = 0L;
-      ndot = nrow = 0;
-      line_bytes = (long)opt.dots_in_line * opt.dot_bytes;
-      if (res)
-	{
-	  if (res >= line_bytes)
+	  /* If bandwidth is not limited, one call to wtimer_elapsed
+	     is sufficient.  */
+	  dltime = wtimer_elapsed (timer);
+	  if (opt.limit_rate)
 	    {
-	      nrow = res / line_bytes;
-	      res %= line_bytes;
-	      logprintf (LOG_VERBOSE,
-			 _("\n          [ skipping %dK ]"),
-			 (int) ((nrow * line_bytes) / 1024));
-	      ndot = 0;
+	      limit_bandwidth (res, dltime - last_dltime);
+	      dltime = wtimer_elapsed (timer);
+	      last_dltime = dltime;
 	    }
+
+	  if (progress)
+	    progress_update (progress, res, dltime);
+	  *len += res;
 	}
-      logprintf (LOG_VERBOSE, "\n%5ldK ->", nrow * line_bytes / 1024);
-    }
-  /* Offset gets incremented by current value.  */
-  offs += res;
-  /* While offset is >= opt.dot_bytes, print dots, taking care to
-     precede every 50th dot with a status message.  */
-  for (; offs >= opt.dot_bytes; offs -= opt.dot_bytes)
-    {
-      if (!(ndot % opt.dot_spacing))
-	logputs (LOG_VERBOSE, " ");
-      any_output = 1;
-      logputs (LOG_VERBOSE, flags == SP_INIT ? "," : ".");
-      ++ndot;
-      if (ndot == opt.dots_in_line)
-	{
-	  ndot = 0;
-	  ++nrow;
-	  if (expected)
-	    print_percentage (nrow * line_bytes, expected);
-	  logprintf (LOG_VERBOSE, "\n%5ldK ->", nrow * line_bytes / 1024);
-	}
+      else
+	break;
     }
-  /* Reenable flushing.  */
-  opt.no_flush = 0;
-  if (any_output)
-    /* Force flush.  #### Oh, what a kludge!  */
-    logflush ();
-  return any_output;
+  if (res < -1)
+    res = -1;
+
+ out:
+  if (progress)
+    progress_finish (progress, dltime);
+  if (elapsed)
+    *elapsed = dltime;
+  wtimer_delete (timer);
+
+  return res;
 }
 
-/* Reset the internal timer.  */
-void
-reset_timer (void)
+/* Return a printed representation of the download rate, as
+   appropriate for the speed.  If PAD is non-zero, strings will be
+   padded to the width of 7 characters (xxxx.xx).  */
+char *
+retr_rate (long bytes, double msecs, int pad)
 {
-#ifndef WINDOWS
-  /* Under Unix, the preferred way to measure the passage of time is
-     through gettimeofday() because of its granularity.  However, on
-     some old or weird systems, gettimeofday() might not be available.
-     There we use the simple time().  */
-# ifdef HAVE_GETTIMEOFDAY
-  struct timeval t;
-  gettimeofday (&t, NULL);
-  internal_secs = t.tv_sec;
-  internal_msecs = t.tv_usec / 1000;
-# else  /* not HAVE_GETTIMEOFDAY */
-  internal_secs = time (NULL);
-  internal_msecs = 0;
-# endif /* not HAVE_GETTIMEOFDAY */
-#else  /* WINDOWS */
-  /* Under Windows, use Windows-specific APIs. */
-  FILETIME ft;
-  SYSTEMTIME st;
-  GetSystemTime(&st);
-  SystemTimeToFileTime(&st,&ft);
-  internal_time.HighPart = ft.dwHighDateTime;
-  internal_time.LowPart = ft.dwLowDateTime;
-#endif /* WINDOWS */
-}
+  static char res[20];
+  static char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };
+  int units = 0;
 
-/* Return the time elapsed from the last call to reset_timer(), in
-   milliseconds.  */
-long
-elapsed_time (void)
-{
-#ifndef WINDOWS
-# ifdef HAVE_GETTIMEOFDAY
-  struct timeval t;
-  gettimeofday (&t, NULL);
-  return ((t.tv_sec - internal_secs) * 1000
-	  + (t.tv_usec / 1000 - internal_msecs));
-# else  /* not HAVE_GETTIMEOFDAY */
-  return 1000 * ((long)time (NULL) - internal_secs);
-# endif /* not HAVE_GETTIMEOFDAY */
-#else  /* WINDOWS */
-  FILETIME ft;
-  SYSTEMTIME st;
-  LARGE_INTEGER li;
-  GetSystemTime(&st);
-  SystemTimeToFileTime(&st,&ft);
-  li.HighPart = ft.dwHighDateTime;
-  li.LowPart = ft.dwLowDateTime;
-  return (long) ((li.QuadPart - internal_time.QuadPart) / 1e4);
-#endif /* WINDOWS */
+  double dlrate = calc_rate (bytes, msecs, &units);
+  sprintf (res, pad ? "%7.2f %s" : "%.2f %s", dlrate, rate_names[units]);
+
+  return res;
 }
 
-/* Print out the appropriate download rate.  Appropriate means that if
-   rate is > 1024 bytes per second, kilobytes are used, and if rate >
-   1024 * 1024 bps, megabytes are used.  */
-char *
-rate (long bytes, long msecs)
+/* Calculate the download rate and trim it as appropriate for the
+   speed.  Appropriate means that if rate is greater than 1K/s,
+   kilobytes are used, and if rate is greater than 1MB/s, megabytes
+   are used.
+
+   UNITS is zero for B/s, one for KB/s, two for MB/s, and three for
+   GB/s.  */
+double
+calc_rate (long bytes, double msecs, int *units)
 {
-  static char res[15];
   double dlrate;
 
-  if (!msecs)
-    ++msecs;
+  assert (msecs >= 0);
+  assert (bytes >= 0);
+
+  if (msecs == 0)
+    /* If elapsed time is exactly zero, it means we're under the
+       granularity of the timer.  This often happens on systems that
+       use time() for the timer.  */
+    msecs = wtimer_granularity ();
+
   dlrate = (double)1000 * bytes / msecs;
-  /* #### Should these strings be translatable?  */
   if (dlrate < 1024.0)
-    sprintf (res, "%.2f B/s", dlrate);
+    *units = 0;
   else if (dlrate < 1024.0 * 1024.0)
-    sprintf (res, "%.2f KB/s", dlrate / 1024.0);
+    *units = 1, dlrate /= 1024.0;
+  else if (dlrate < 1024.0 * 1024.0 * 1024.0)
+    *units = 2, dlrate /= (1024.0 * 1024.0);
   else
-    sprintf (res, "%.2f MB/s", dlrate / (1024.0 * 1024.0));
-  return res;
+    /* Maybe someone will need this one day.  More realistically, it
+       will get tickled by buggy timers. */
+    *units = 3, dlrate /= (1024.0 * 1024.0 * 1024.0);
+
+  return dlrate;
 }
 
-#define USE_PROXY_P(u) (opt.use_proxy && getproxy((u)->proto)		\
-			&& no_proxy_match((u)->host,			\
-					  (const char **)opt.no_proxy))
+/* Maximum number of allowed redirections.  20 was chosen as a
+   "reasonable" value, which is low enough to not cause havoc, yet
+   high enough to guarantee that normal retrievals will not be hurt by
+   the check.  */
+
+#define MAX_REDIRECTIONS 20
+
+#define SUSPEND_POST_DATA do {			\
+  post_data_suspended = 1;			\
+  saved_post_data = opt.post_data;		\
+  saved_post_file_name = opt.post_file_name;	\
+  opt.post_data = NULL;				\
+  opt.post_file_name = NULL;			\
+} while (0)
+
+#define RESTORE_POST_DATA do {				\
+  if (post_data_suspended)				\
+    {							\
+      opt.post_data = saved_post_data;			\
+      opt.post_file_name = saved_post_file_name;	\
+      post_data_suspended = 0;				\
+    }							\
+} while (0)
 
 /* Retrieve the given URL.  Decides which loop to call -- HTTP, FTP,
-   or simply copy it with file:// (#### the latter not yet
-   implemented!).  */
+   FTP, proxy, etc.  */
+
+/* #### This function should be rewritten so it doesn't return from
+   multiple points. */
+
 uerr_t
 retrieve_url (const char *origurl, char **file, char **newloc,
 	      const char *refurl, int *dt)
@@ -320,10 +321,15 @@ retrieve_url (const char *origurl, char **file, char **newloc,
   uerr_t result;
   char *url;
   int location_changed, dummy;
-  int local_use_proxy;
   char *mynewloc, *proxy;
-  struct urlinfo *u;
-  slist *redirections;
+  struct url *u, *proxy_url;
+  int up_error_code;		/* url parse error code */
+  char *local_file;
+  int redirection_count = 0;
+
+  int post_data_suspended = 0;
+  char *saved_post_data = NULL;
+  char *saved_post_file_name = NULL;
 
   /* If dt is NULL, just ignore it.  */
   if (!dt)
@@ -334,188 +340,185 @@ retrieve_url (const char *origurl, char **file, char **newloc,
   if (file)
     *file = NULL;
 
-  redirections = NULL;
-
-  u = newurl ();
-  /* Parse the URL. */
-  result = parseurl (url, u, 0);
-  if (result != URLOK)
+  u = url_parse (url, &up_error_code);
+  if (!u)
     {
-      logprintf (LOG_NOTQUIET, "%s: %s.\n", url, uerrmsg (result));
-      freeurl (u, 1);
-      free_slist (redirections);
-      free (url);
-      return result;
+      logprintf (LOG_NOTQUIET, "%s: %s.\n", url, url_error (up_error_code));
+      xfree (url);
+      return URLERROR;
     }
 
+  if (!refurl)
+    refurl = opt.referer;
+
  redirected:
 
-  /* Set the referer.  */
-  if (refurl)
-    u->referer = xstrdup (refurl);
-  else
-    {
-      if (opt.referer)
-	u->referer = xstrdup (opt.referer);
-      else
-	u->referer = NULL;
-    }
+  result = NOCONERROR;
+  mynewloc = NULL;
+  local_file = NULL;
+  proxy_url = NULL;
 
-  local_use_proxy = USE_PROXY_P (u);
-  if (local_use_proxy)
+  proxy = getproxy (u);
+  if (proxy)
     {
-      struct urlinfo *pu = newurl ();
-
-      /* Copy the original URL to new location.  */
-      memcpy (pu, u, sizeof (*u));
-      pu->proxy = NULL; /* A minor correction :) */
-      /* Initialize u to nil.  */
-      memset (u, 0, sizeof (*u));
-      u->proxy = pu;
-      /* Get the appropriate proxy server, appropriate for the
-	 current protocol.  */
-      proxy = getproxy (pu->proto);
-      if (!proxy)
+      /* Parse the proxy URL.  */
+      proxy_url = url_parse (proxy, &up_error_code);
+      if (!proxy_url)
 	{
-	  logputs (LOG_NOTQUIET, _("Could not find proxy host.\n"));
-	  freeurl (u, 1);
-	  free_slist (redirections);
-	  free (url);
+	  logprintf (LOG_NOTQUIET, _("Error parsing proxy URL %s: %s.\n"),
+		     proxy, url_error (up_error_code));
+	  xfree (url);
+	  RESTORE_POST_DATA;
 	  return PROXERR;
 	}
-      /* Parse the proxy URL.  */
-      result = parseurl (proxy, u, 0);
-      if (result != URLOK || u->proto != URLHTTP)
+      if (proxy_url->scheme != SCHEME_HTTP && proxy_url->scheme != u->scheme)
 	{
-	  if (u->proto == URLHTTP)
-	    logprintf (LOG_NOTQUIET, "Proxy %s: %s.\n", proxy, uerrmsg(result));
-	  else
-	    logprintf (LOG_NOTQUIET, _("Proxy %s: Must be HTTP.\n"), proxy);
-	  freeurl (u, 1);
-	  free_slist (redirections);
-	  free (url);
+	  logprintf (LOG_NOTQUIET, _("Error in proxy URL %s: Must be HTTP.\n"), proxy);
+	  url_free (proxy_url);
+	  xfree (url);
+	  RESTORE_POST_DATA;
 	  return PROXERR;
 	}
-      u->proto = URLHTTP;
     }
 
-  assert (u->proto != URLFILE);	/* #### Implement me!  */
-  mynewloc = NULL;
-
-  if (u->proto == URLHTTP)
-    result = http_loop (u, &mynewloc, dt);
-  else if (u->proto == URLFTP)
+  if (u->scheme == SCHEME_HTTP
+#ifdef HAVE_SSL
+      || u->scheme == SCHEME_HTTPS
+#endif
+      || (proxy_url && proxy_url->scheme == SCHEME_HTTP))
+    {
+      result = http_loop (u, &mynewloc, &local_file, refurl, dt, proxy_url);
+    }
+  else if (u->scheme == SCHEME_FTP)
     {
       /* If this is a redirection, we must not allow recursive FTP
 	 retrieval, so we save recursion to oldrec, and restore it
 	 later.  */
       int oldrec = opt.recursive;
-      if (redirections)
+      if (redirection_count)
 	opt.recursive = 0;
-      result = ftp_loop (u, dt);
+      result = ftp_loop (u, dt, proxy_url);
       opt.recursive = oldrec;
+
       /* There is a possibility of having HTTP being redirected to
 	 FTP.  In these cases we must decide whether the text is HTML
-	 according to the suffix.  The HTML suffixes are `.html' and
-	 `.htm', case-insensitive.
-
-	 #### All of this is, of course, crap.  These types should be
-	 determined through mailcap.  */
-      if (redirections && u->local && (u->proto == URLFTP ))
+	 according to the suffix.  The HTML suffixes are `.html',
+	 `.htm' and a few others, case-insensitive.  */
+      if (redirection_count && local_file && u->scheme == SCHEME_FTP)
 	{
-	  char *suf = suffix (u->local);
-	  if (suf && (!strcasecmp (suf, "html") || !strcasecmp (suf, "htm")))
+	  if (has_html_suffix_p (local_file))
 	    *dt |= TEXTHTML;
-	  FREE_MAYBE (suf);
 	}
     }
+
+  if (proxy_url)
+    {
+      url_free (proxy_url);
+      proxy_url = NULL;
+    }
+
   location_changed = (result == NEWLOCATION);
   if (location_changed)
     {
       char *construced_newloc;
-      uerr_t newloc_result;
-      struct urlinfo *newloc_struct;
+      struct url *newloc_parsed;
 
       assert (mynewloc != NULL);
 
+      if (local_file)
+	xfree (local_file);
+
       /* The HTTP specs only allow absolute URLs to appear in
 	 redirects, but a ton of boneheaded webservers and CGIs out
 	 there break the rules and use relative URLs, and popular
 	 browsers are lenient about this, so wget should be too. */
-      construced_newloc = url_concat (url, mynewloc);
-      free (mynewloc);
+      construced_newloc = uri_merge (url, mynewloc);
+      xfree (mynewloc);
       mynewloc = construced_newloc;
 
       /* Now, see if this new location makes sense. */
-      newloc_struct = newurl ();
-      newloc_result = parseurl (mynewloc, newloc_struct, 1);
-      if (newloc_result != URLOK)
+      newloc_parsed = url_parse (mynewloc, &up_error_code);
+      if (!newloc_parsed)
 	{
-	  logprintf (LOG_NOTQUIET, "%s: %s.\n", mynewloc, uerrmsg (newloc_result));
-	  freeurl (newloc_struct, 1);
-	  freeurl (u, 1);
-	  free_slist (redirections);
-	  free (url);
-	  free (mynewloc);
+	  logprintf (LOG_NOTQUIET, "%s: %s.\n", mynewloc,
+		     url_error (up_error_code));
+	  url_free (u);
+	  xfree (url);
+	  xfree (mynewloc);
+	  RESTORE_POST_DATA;
 	  return result;
 	}
 
-      /* Now mynewloc will become newloc_struct->url, because if the
+      /* Now mynewloc will become newloc_parsed->url, because if the
          Location contained relative paths like .././something, we
          don't want that propagating as url.  */
-      free (mynewloc);
-      mynewloc = xstrdup (newloc_struct->url);
+      xfree (mynewloc);
+      mynewloc = xstrdup (newloc_parsed->url);
 
-      /* Check for redirection to back to itself.  */
-      if (!strcmp (u->url, newloc_struct->url))
+      /* Check for max. number of redirections.  */
+      if (++redirection_count > MAX_REDIRECTIONS)
 	{
-	  logprintf (LOG_NOTQUIET, _("%s: Redirection to itself.\n"),
-		     mynewloc);
-	  freeurl (newloc_struct, 1);
-	  freeurl (u, 1);
-	  free_slist (redirections);
-	  free (url);
-	  free (mynewloc);
+	  logprintf (LOG_NOTQUIET, _("%d redirections exceeded.\n"),
+		     MAX_REDIRECTIONS);
+	  url_free (newloc_parsed);
+	  url_free (u);
+	  xfree (url);
+	  xfree (mynewloc);
+	  RESTORE_POST_DATA;
 	  return WRONGCODE;
 	}
 
-      /* The new location is OK.  Let's check for redirection cycle by
-         peeking through the history of redirections. */
-      if (in_slist (redirections, newloc_struct->url))
-	{
-	  logprintf (LOG_NOTQUIET, _("%s: Redirection cycle detected.\n"),
-		     mynewloc);
-	  freeurl (newloc_struct, 1);
-	  freeurl (u, 1);
-	  free_slist (redirections);
-	  free (url);
-	  free (mynewloc);
-	  return WRONGCODE;
-	}
+      xfree (url);
+      url = mynewloc;
+      url_free (u);
+      u = newloc_parsed;
 
-      redirections = add_slist (redirections, newloc_struct->url, NOSORT);
+      /* If we're being redirected from POST, we don't want to POST
+	 again.  Many requests answer POST with a redirection to an
+	 index page; that redirection is clearly a GET.  We "suspend"
+	 POST data for the duration of the redirections, and restore
+	 it when we're done. */
+      if (!post_data_suspended)
+	SUSPEND_POST_DATA;
 
-      free (url);
-      url = mynewloc;
-      freeurl (u, 1);
-      u = newloc_struct;
       goto redirected;
     }
 
+  if (local_file)
+    {
+      if (*dt & RETROKF)
+	{
+	  register_download (u->url, local_file);
+	  if (redirection_count && 0 != strcmp (origurl, u->url))
+	    register_redirection (origurl, u->url);
+	  if (*dt & TEXTHTML)
+	    register_html (u->url, local_file);
+	}
+    }
+
   if (file)
+    *file = local_file ? local_file : NULL;
+  else
+    FREE_MAYBE (local_file);
+
+  url_free (u);
+
+  if (redirection_count)
     {
-      if (u->local)
-	*file = xstrdup (u->local);
+      if (newloc)
+	*newloc = url;
       else
-	*file = NULL;
+	xfree (url);
     }
-  freeurl (u, 1);
-  free_slist (redirections);
-
-  if (newloc)
-    *newloc = url;
   else
-    free (url);
+    {
+      if (newloc)
+	*newloc = NULL;
+      xfree (url);
+    }
+
+  ++global_download_count;
+  RESTORE_POST_DATA;
 
   return result;
 }
@@ -529,29 +532,30 @@ uerr_t
 retrieve_from_file (const char *file, int html, int *count)
 {
   uerr_t status;
-  urlpos *url_list, *cur_url;
+  struct urlpos *url_list, *cur_url;
 
-  /* If spider-mode is on, we do not want get_urls_html barfing
-     errors on baseless links.  */
-  url_list = (html ? get_urls_html (file, NULL, opt.spider, FALSE)
+  url_list = (html ? get_urls_html (file, NULL, NULL)
 	      : get_urls_file (file));
   status = RETROK;             /* Suppose everything is OK.  */
   *count = 0;                  /* Reset the URL count.  */
-  recursive_reset ();
+
   for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count)
     {
-      char *filename, *new_file;
+      char *filename = NULL, *new_file = NULL;
       int dt;
 
+      if (cur_url->ignore_when_downloading)
+	continue;
+
       if (downloaded_exceeds_quota ())
 	{
 	  status = QUOTEXC;
 	  break;
 	}
-      status = retrieve_url (cur_url->url, &filename, &new_file, NULL, &dt);
-      if (opt.recursive && status == RETROK && (dt & TEXTHTML))
-	status = recursive_retrieve (filename, new_file ? new_file
-				                        : cur_url->url);
+      if (opt.recursive && cur_url->url->scheme != SCHEME_FTP)
+	status = retrieve_tree (cur_url->url->url);
+      else
+	status = retrieve_url (cur_url->url->url, &filename, &new_file, NULL, &dt);
 
       if (filename && opt.delete_after && file_exists_p (filename))
 	{
@@ -614,3 +618,53 @@ downloaded_exceeds_quota (void)
 
   return opt.downloaded > opt.quota;
 }
+
+/* If opt.wait or opt.waitretry are specified, and if certain
+   conditions are met, sleep the appropriate number of seconds.  See
+   the documentation of --wait and --waitretry for more information.
+
+   COUNT is the count of current retrieval, beginning with 1. */
+
+void
+sleep_between_retrievals (int count)
+{
+  static int first_retrieval = 1;
+
+  if (first_retrieval)
+    {
+      /* Don't sleep before the very first retrieval. */
+      first_retrieval = 0;
+      return;
+    }
+
+  if (opt.waitretry && count > 1)
+    {
+      /* If opt.waitretry is specified and this is a retry, wait for
+	 COUNT-1 number of seconds, or for opt.waitretry seconds.  */
+      if (count <= opt.waitretry)
+	sleep (count - 1);
+      else
+	sleep (opt.waitretry);
+    }
+  else if (opt.wait)
+    {
+      if (!opt.random_wait || count > 1)
+	/* If random-wait is not specified, or if we are sleeping
+	   between retries of the same download, sleep the fixed
+	   interval.  */
+	sleep (opt.wait);
+      else
+	{
+	  /* Sleep a random amount of time averaging in opt.wait
+	     seconds.  The sleeping amount ranges from 0 to
+	     opt.wait*2, inclusive.  */
+	  int waitsecs = random_number (opt.wait * 2 + 1);
+
+	  DEBUGP (("sleep_between_retrievals: norm=%ld,fuzz=%ld,sleep=%d\n",
+		   opt.wait, waitsecs - opt.wait, waitsecs));
+
+	  if (waitsecs)
+	    sleep (waitsecs);
+	}
+    }
+}