X-Git-Url: http://sjero.net/git/?a=blobdiff_plain;f=src%2Fretr.c;h=5d9795ad67ac7177163f96d1dff28f2eaa6bb9c7;hb=d9fea91a0a319e348adb504bd3edff148ff3d8a0;hp=637f9316f618a9ab04f6bb7122a7ec2f1133abe7;hpb=e2e9b753e47efb8f228b9b14f04f307e4f2489b3;p=wget

diff --git a/src/retr.c b/src/retr.c
index 637f9316..5d9795ad 100644
--- a/src/retr.c
+++ b/src/retr.c
@@ -84,13 +84,13 @@ limit_bandwidth_reset (void)
 }
 
 /* Limit the bandwidth by pausing the download for an amount of time.
-   BYTES is the number of bytes received from the network, and DELTA
-   is the number of milliseconds it took to receive them.  */
+   BYTES is the number of bytes received from the network, and TIMER
+   is the timer that started at the beginning of download.  */
 
 static void
-limit_bandwidth (long bytes, double *dltime, struct wget_timer *timer)
+limit_bandwidth (long bytes, struct wget_timer *timer)
 {
-  double delta_t = *dltime - limit_data.chunk_start;
+  double delta_t = wtimer_read (timer) - limit_data.chunk_start;
   double expected;
 
   limit_data.chunk_bytes += bytes;
@@ -113,32 +113,27 @@ limit_bandwidth (long bytes, double *dltime, struct wget_timer *timer)
       DEBUGP (("\nsleeping %.2f ms for %ld bytes, adjust %.2f ms\n",
 	       slp, limit_data.chunk_bytes, limit_data.sleep_adjust));
 
-      t0 = *dltime;
-      usleep ((unsigned long) (1000 * slp));
-      t1 = wtimer_elapsed (timer);
+      t0 = wtimer_read (timer);
+      xsleep (slp / 1000);
+      wtimer_update (timer);
+      t1 = wtimer_read (timer);
 
       /* Due to scheduling, we probably slept slightly longer (or
 	 shorter) than desired.  Calculate the difference between the
 	 desired and the actual sleep, and adjust the next sleep by
 	 that amount.  */
       limit_data.sleep_adjust = slp - (t1 - t0);
-
-      /* Since we've called wtimer_elapsed, we might as well update
-	 the caller's dltime. */
-      *dltime = t1;
     }
 
   limit_data.chunk_bytes = 0;
-  limit_data.chunk_start = *dltime;
+  limit_data.chunk_start = wtimer_read (timer);
 }
 
 #define MIN(i, j) ((i) <= (j) ? (i) : (j))
 
 /* Reads the contents of file descriptor FD, until it is closed, or a
    read error occurs.  The data is read in 8K chunks, and stored to
-   stream fp, which should have been open for writing.  If BUF is
-   non-NULL and its file descriptor is equal to FD, flush RBUF first.
-   This function will *not* use the rbuf_* functions!
+   stream fp, which should have been open for writing.
 
    The EXPECTED argument is passed to show_progress() unchanged, but
    otherwise ignored.
@@ -150,52 +145,41 @@ limit_bandwidth (long bytes, double *dltime, struct wget_timer *timer)
 
    The function exits and returns codes of 0, -1 and -2 if the
    connection was closed, there was a read error, or if it could not
-   write to the output stream, respectively.
+   write to the output stream, respectively.  */
 
-   IMPORTANT: The function flushes the contents of the buffer in
-   rbuf_flush() before actually reading from fd.  If you wish to read
-   from fd immediately, flush or discard the buffer.  */
 int
-get_contents (int fd, FILE *fp, long *len, long restval, long expected,
-	      struct rbuf *rbuf, int use_expected, double *elapsed)
+fd_read_body (int fd, FILE *out, long *len, long restval, long expected,
+	      int use_expected, double *elapsed)
 {
   int res = 0;
 
   static char dlbuf[16384];
   int dlbufsize = sizeof (dlbuf);
 
-  void *progress = NULL;
   struct wget_timer *timer = wtimer_allocate ();
-  double dltime = 0;
+  double last_successful_read_tm;
+
+  /* The progress gauge, set according to the user preferences. */
+  void *progress = NULL;
+
+  /* Non-zero if the progress gauge is interactive, i.e. if it can
+     continually update the display.  When true, smaller timeout
+     values are used so that the gauge can update the display when
+     data arrives slowly. */
+  int progress_interactive = 0;
 
   *len = restval;
 
   if (opt.verbose)
-    progress = progress_create (restval, expected);
-
-  if (rbuf && RBUF_FD (rbuf) == fd)
     {
-      int sz = 0;
-      while ((res = rbuf_flush (rbuf, dlbuf, sizeof (dlbuf))) != 0)
-	{
-	  fwrite (dlbuf, 1, res, fp);
-	  *len += res;
-	  sz += res;
-	}
-      if (sz)
-	fflush (fp);
-      if (ferror (fp))
-	{
-	  res = -2;
-	  goto out;
-	}
-      if (progress)
-	progress_update (progress, sz, 0);
+      progress = progress_create (restval, expected);
+      progress_interactive = progress_interactive_p (progress);
     }
 
   if (opt.limit_rate)
     limit_bandwidth_reset ();
   wtimer_reset (timer);
+  last_successful_read_tm = 0;
 
   /* Use a smaller buffer for low requested bandwidths.  For example,
      with --limit-rate=2k, it doesn't make sense to slurp in 16K of
@@ -213,35 +197,58 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
     {
       int amount_to_read = (use_expected
 			    ? MIN (expected - *len, dlbufsize) : dlbufsize);
-#ifdef HAVE_SSL
-      if (rbuf->ssl!=NULL)
-	res = ssl_iread (rbuf->ssl, dlbuf, amount_to_read);
-      else
-#endif /* HAVE_SSL */
-	res = iread (fd, dlbuf, amount_to_read);
+      double tmout = opt.read_timeout;
+      if (progress_interactive)
+	{
+	  double waittm;
+	  /* For interactive progress gauges, always specify a ~1s
+	     timeout, so that the gauge can be updated regularly even
+	     when the data arrives very slowly or stalls.  */
+	  tmout = 0.95;
+	  waittm = (wtimer_read (timer) - last_successful_read_tm) / 1000;
+	  if (waittm + tmout > opt.read_timeout)
+	    {
+	      /* Don't allow waiting time to exceed read timeout. */
+	      tmout = opt.read_timeout - waittm;
+	      if (tmout < 0)
+		{
+		  /* We've already exceeded the timeout. */
+		  res = -1, errno = ETIMEDOUT;
+		  break;
+		}
+	    }
+	}
+      res = fd_read (fd, dlbuf, amount_to_read, tmout);
 
-      if (res <= 0)
+      if (res == 0 || (res < 0 && errno != ETIMEDOUT))
 	break;
+      else if (res < 0)
+	res = 0;		/* timeout */
 
-      fwrite (dlbuf, 1, res, fp);
-      /* Always flush the contents of the network packet.  This should
-	 not hinder performance: fast downloads will be received in
-	 16K chunks (which stdio would write out anyway), and slow
-	 downloads won't be limited with disk performance.  */
-      fflush (fp);
-      if (ferror (fp))
+      wtimer_update (timer);
+      if (res > 0)
 	{
-	  res = -2;
-	  goto out;
+	  fwrite (dlbuf, 1, res, out);
+	  /* Always flush the contents of the network packet.  This
+	     should not hinder performance: fast downloads will be
+	     received in 16K chunks (which stdio would write out
+	     anyway), and slow downloads won't be limited by disk
+	     performance.  */
+	  fflush (out);
+	  if (ferror (out))
+	    {
+	      res = -2;
+	      goto out;
+	    }
+	  last_successful_read_tm = wtimer_read (timer);
 	}
 
-      dltime = wtimer_elapsed (timer);
       if (opt.limit_rate)
-	limit_bandwidth (res, &dltime, timer);
+	limit_bandwidth (res, timer);
 
       *len += res;
       if (progress)
-	progress_update (progress, res, dltime);
+	progress_update (progress, res, wtimer_read (timer));
 #ifdef WINDOWS
       if (use_expected && expected > 0)
 	ws_percenttitle (100.0 * (double)(*len) / (double)expected);
@@ -252,14 +259,165 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
 
  out:
   if (progress)
-    progress_finish (progress, dltime);
+    progress_finish (progress, wtimer_read (timer));
   if (elapsed)
-    *elapsed = dltime;
+    *elapsed = wtimer_read (timer);
   wtimer_delete (timer);
 
   return res;
 }
 
+typedef const char *(*finder_t) PARAMS ((const char *, int, int));
+
+/* Driver for fd_read_line and fd_read_head: keeps reading data until
+   a terminator (as decided by FINDER) occurs in the data.  The trick
+   is that the data is first peeked at, and only then actually read.
+   That way the data after the terminator is never read.  */
+
+static char *
+fd_read_until (int fd, finder_t finder, int bufsize)
+{
+  int size = bufsize, tail = 0;
+  char *buf = xmalloc (size);
+
+  while (1)
+    {
+      const char *end;
+      int pklen, rdlen, remain;
+
+      /* First, peek at the available data. */
+
+      pklen = fd_peek (fd, buf + tail, size - tail, -1);
+      if (pklen < 0)
+	{
+	  xfree (buf);
+	  return NULL;
+	}
+      end = finder (buf, tail, pklen);
+      if (end)
+	{
+	  /* The data contains the terminator: we'll read the data up
+	     to the end of the terminator.  */
+	  remain = end - (buf + tail);
+	  /* Note +1 for trailing \0. */
+	  if (size < tail + remain + 1)
+	    {
+	      size = tail + remain + 1;
+	      buf = xrealloc (buf, size);
+	    }
+	}
+      else
+	/* No terminator: simply read the data we know is (or should
+	   be) available.  */
+	remain = pklen;
+
+      /* Now, read the data.  Note that we make no assumptions about
+	 how much data we'll get.  (Some TCP stacks are notorious for
+	 read returning less data than the previous MSG_PEEK.)  */
+
+      rdlen = fd_read (fd, buf + tail, remain, 0);
+      if (rdlen < 0)
+	{
+	  xfree_null (buf);
+	  return NULL;
+	}
+      if (rdlen == 0)
+	{
+	  if (tail == 0)
+	    {
+	      /* EOF without anything having been read */
+	      xfree (buf);
+	      errno = 0;
+	      return NULL;
+	    }
+	  /* Return what we received so far. */
+	  if (size < tail + 1)
+	    {
+	      size = tail + 1;	/* expand the buffer to receive the
+				   terminating \0 */
+	      buf = xrealloc (buf, size);
+	    }
+	  buf[tail] = '\0';
+	  return buf;
+	}
+      tail += rdlen;
+      if (end && rdlen == remain)
+	{
+	  /* The end was seen and the data read -- we got what we came
+	     for.  */
+	  buf[tail] = '\0';
+	  return buf;
+	}
+
+      /* Keep looping until all the data arrives. */
+
+      if (tail == size)
+	{
+	  size <<= 1;
+	  buf = xrealloc (buf, size);
+	}
+    }
+}
+
+static const char *
+line_terminator (const char *buf, int tail, int peeklen)
+{
+  const char *p = memchr (buf + tail, '\n', peeklen);
+  if (p)
+    /* p+1 because we want the line to include '\n' */
+    return p + 1;
+  return NULL;
+}
+
+/* Read one line from FD and return it.  The line is allocated using
+   malloc.
+
+   If an error occurs, or if no data can be read, NULL is returned.
+   In the former case errno indicates the error condition, and in the
+   latter case, errno is NULL.  */
+
+char *
+fd_read_line (int fd)
+{
+  return fd_read_until (fd, line_terminator, 128);
+}
+
+static const char *
+head_terminator (const char *buf, int tail, int peeklen)
+{
+  const char *start, *end;
+  if (tail < 4)
+    start = buf;
+  else
+    start = buf + tail - 4;
+  end = buf + tail + peeklen;
+
+  for (; start < end - 1; start++)
+    if (*start == '\n')
+      {
+	if (start < end - 2
+	    && start[1] == '\r'
+	    && start[2] == '\n')
+	  return start + 3;
+	if (start[1] == '\n')
+	  return start + 2;
+      }
+  return NULL;
+}
+
+/* Read the request head from FD and return it.  The chunk of data is
+   allocated using malloc.
+
+   If an error occurs, or if no data can be read, NULL is returned.
+   In the former case errno indicates the error condition, and in the
+   latter case, errno is NULL.  */
+
+char *
+fd_read_head (int fd)
+{
+  return fd_read_until (fd, head_terminator, 512);
+}
+
 /* Return a printed representation of the download rate, as
    appropriate for the speed.  If PAD is non-zero, strings will be
    padded to the width of 7 characters (xxxx.xx).  */
@@ -642,9 +800,9 @@ sleep_between_retrievals (int count)
       /* If opt.waitretry is specified and this is a retry, wait for
 	 COUNT-1 number of seconds, or for opt.waitretry seconds.  */
       if (count <= opt.waitretry)
-	sleep (count - 1);
+	xsleep (count - 1);
       else
-	usleep (1000000L * opt.waitretry);
+	xsleep (opt.waitretry);
     }
   else if (opt.wait)
     {
@@ -652,7 +810,7 @@ sleep_between_retrievals (int count)
 	/* If random-wait is not specified, or if we are sleeping
 	   between retries of the same download, sleep the fixed
 	   interval.  */
-	usleep (1000000L * opt.wait);
+	xsleep (opt.wait);
       else
 	{
 	  /* Sleep a random amount of time averaging in opt.wait
@@ -661,7 +819,7 @@ sleep_between_retrievals (int count)
 	  double waitsecs = 2 * opt.wait * random_float ();
 	  DEBUGP (("sleep_between_retrievals: avg=%f,sleep=%f\n",
 		   opt.wait, waitsecs));
-	  usleep (1000000L * waitsecs);
+	  xsleep (waitsecs);
 	}
     }
 }