[svn] New function xsleep that resumes sleeps interrupted by signals

[wget] / src / retr.c

diff --git a/src/retr.c b/src/retr.c
index 1af9a1b899ce6723cb328b4b8a0bfa5c49ac7f3e..c3da16417072f1400268443d126fe4428feb9fdb 100644 (file)
--- a/src/retr.c
+++ b/src/retr.c
@@ -66,6 +66,9 @@ extern int errno;
 /* See the comment in gethttp() why this is needed. */
 int global_download_count;
 
+/* Total size of downloaded files.  Used to enforce quota.  */
+LARGE_INT total_downloaded_bytes;
+
 \f
 static struct {
   long chunk_bytes;
@@ -111,7 +114,7 @@ limit_bandwidth (long bytes, double *dltime, struct wget_timer *timer)
               slp, limit_data.chunk_bytes, limit_data.sleep_adjust));
 
       t0 = *dltime;
-      usleep ((unsigned long) (1000 * slp));
+      xsleep (slp / 1000);
       t1 = wtimer_elapsed (timer);
 
       /* Due to scheduling, we probably slept slightly longer (or
@@ -194,8 +197,10 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
     limit_bandwidth_reset ();
   wtimer_reset (timer);
 
-  /* If we're limiting the download, set our buffer size to the
-     limit.  */
+  /* Use a smaller buffer for low requested bandwidths.  For example,
+     with --limit-rate=2k, it doesn't make sense to slurp in 16K of
+     data and then sleep for 8s.  With buffer size equal to the limit,
+     we never have to sleep for more than one second.  */
   if (opt.limit_rate && opt.limit_rate < dlbufsize)
     dlbufsize = opt.limit_rate;
 
@@ -234,9 +239,13 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
       if (opt.limit_rate)
        limit_bandwidth (res, &dltime, timer);
 
+      *len += res;
       if (progress)
        progress_update (progress, res, dltime);
-      *len += res;
+#ifdef WINDOWS
+      if (use_expected && expected > 0)
+       ws_percenttitle (100.0 * (double)(*len) / (double)expected);
+#endif
     }
   if (res < -1)
     res = -1;
@@ -351,9 +360,12 @@ retrieve_url (const char *origurl, char **file, char **newloc,
   char *saved_post_data = NULL;
   char *saved_post_file_name = NULL;
 
-  /* If dt is NULL, just ignore it.  */
+  /* If dt is NULL, use local storage.  */
   if (!dt)
-    dt = &dummy;
+    {
+      dt = &dummy;
+      dummy = 0;
+    }
   url = xstrdup (origurl);
   if (newloc)
     *newloc = NULL;
@@ -519,7 +531,7 @@ retrieve_url (const char *origurl, char **file, char **newloc,
   if (file)
     *file = local_file ? local_file : NULL;
   else
-    FREE_MAYBE (local_file);
+    xfree_null (local_file);
 
   url_free (u);
 
@@ -547,7 +559,8 @@ retrieve_url (const char *origurl, char **file, char **newloc,
    them.  If HTML is non-zero, treat the file as HTML, and construct
    the URLs accordingly.
 
-   If opt.recursive is set, call recursive_retrieve() for each file.  */
+   If opt.recursive is set, call retrieve_tree() for each file.  */
+
 uerr_t
 retrieve_from_file (const char *file, int html, int *count)
 {
@@ -567,12 +580,13 @@ retrieve_from_file (const char *file, int html, int *count)
       if (cur_url->ignore_when_downloading)
        continue;
 
-      if (downloaded_exceeds_quota ())
+      if (opt.quota && total_downloaded_bytes > opt.quota)
        {
          status = QUOTEXC;
          break;
        }
-      if (opt.recursive && cur_url->url->scheme != SCHEME_FTP)
+      if ((opt.recursive || opt.page_requisites)
+         && cur_url->url->scheme != SCHEME_FTP)
        status = retrieve_tree (cur_url->url->url);
       else
        status = retrieve_url (cur_url->url->url, &filename, &new_file, NULL, &dt);
@@ -587,8 +601,8 @@ retrieve_from_file (const char *file, int html, int *count)
          dt &= ~RETROKF;
        }
 
-      FREE_MAYBE (new_file);
-      FREE_MAYBE (filename);
+      xfree_null (new_file);
+      xfree_null (filename);
     }
 
   /* Free the linked list of URL-s.  */
@@ -605,40 +619,6 @@ printwhat (int n1, int n2)
   logputs (LOG_VERBOSE, (n1 == n2) ? _("Giving up.\n\n") : _("Retrying.\n\n"));
 }
 
-/* Increment opt.downloaded by BY_HOW_MUCH.  If an overflow occurs,
-   set opt.downloaded_overflow to 1. */
-void
-downloaded_increase (unsigned long by_how_much)
-{
-  VERY_LONG_TYPE old;
-  if (opt.downloaded_overflow)
-    return;
-  old = opt.downloaded;
-  opt.downloaded += by_how_much;
-  if (opt.downloaded < old)    /* carry flag, where are you when I
-                                   need you? */
-    {
-      /* Overflow. */
-      opt.downloaded_overflow = 1;
-      opt.downloaded = ~((VERY_LONG_TYPE)0);
-    }
-}
-
-/* Return non-zero if the downloaded amount of bytes exceeds the
-   desired quota.  If quota is not set or if the amount overflowed, 0
-   is returned. */
-int
-downloaded_exceeds_quota (void)
-{
-  if (!opt.quota)
-    return 0;
-  if (opt.downloaded_overflow)
-    /* We don't really know.  (Wildly) assume not. */
-    return 0;
-
-  return opt.downloaded > opt.quota;
-}
-
 /* If opt.wait or opt.waitretry are specified, and if certain
    conditions are met, sleep the appropriate number of seconds.  See
    the documentation of --wait and --waitretry for more information.
@@ -662,9 +642,9 @@ sleep_between_retrievals (int count)
       /* If opt.waitretry is specified and this is a retry, wait for
         COUNT-1 number of seconds, or for opt.waitretry seconds.  */
       if (count <= opt.waitretry)
-       sleep (count - 1);
+       xsleep (count - 1);
       else
-       usleep (1000000L * opt.waitretry);
+       xsleep (opt.waitretry);
     }
   else if (opt.wait)
     {
@@ -672,7 +652,7 @@ sleep_between_retrievals (int count)
        /* If random-wait is not specified, or if we are sleeping
           between retries of the same download, sleep the fixed
           interval.  */
-       usleep (1000000L * opt.wait);
+       xsleep (opt.wait);
       else
        {
          /* Sleep a random amount of time averaging in opt.wait
@@ -681,7 +661,7 @@ sleep_between_retrievals (int count)
          double waitsecs = 2 * opt.wait * random_float ();
          DEBUGP (("sleep_between_retrievals: avg=%f,sleep=%f\n",
                   opt.wait, waitsecs));
-         usleep (1000000L * waitsecs);
+         xsleep (waitsecs);
        }
     }
 }
@@ -695,7 +675,7 @@ free_urlpos (struct urlpos *l)
       struct urlpos *next = l->next;
       if (l->url)
        url_free (l->url);
-      FREE_MAYBE (l->local_name);
+      xfree_null (l->local_name);
       xfree (l);
       l = next;
     }