extern int errno;
#endif
-/* See the comment in gethttp() why this is needed. */
-int global_download_count;
-
/* Total size of downloaded files. Used to enforce quota. */
LARGE_INT total_downloaded_bytes;
+/* If non-NULL, the stream to which output should be written. This
+ stream is initialized when `-O' is used. */
+FILE *output_stream;
+
+/* Whether output_document is a regular file we can manipulate,
+ i.e. not `-' or a device file. */
+int output_stream_regular;
\f
static struct {
- long chunk_bytes;
+ wgint chunk_bytes;
double chunk_start;
double sleep_adjust;
} limit_data;
is the timer that started at the beginning of download. */
static void
-limit_bandwidth (long bytes, struct wget_timer *timer)
+limit_bandwidth (wgint bytes, struct wget_timer *timer)
{
double delta_t = wtimer_read (timer) - limit_data.chunk_start;
double expected;
double t0, t1;
if (slp < 200)
{
- DEBUGP (("deferring a %.2f ms sleep (%ld/%.2f).\n",
- slp, limit_data.chunk_bytes, delta_t));
+ DEBUGP (("deferring a %.2f ms sleep (%s/%.2f).\n",
+ slp, number_to_static_string (limit_data.chunk_bytes),
+ delta_t));
return;
}
- DEBUGP (("\nsleeping %.2f ms for %ld bytes, adjust %.2f ms\n",
- slp, limit_data.chunk_bytes, limit_data.sleep_adjust));
+ DEBUGP (("\nsleeping %.2f ms for %s bytes, adjust %.2f ms\n",
+ slp, number_to_static_string (limit_data.chunk_bytes),
+ limit_data.sleep_adjust));
t0 = wtimer_read (timer);
xsleep (slp / 1000);
# define MIN(i, j) ((i) <= (j) ? (i) : (j))
#endif
+/* Write data in BUF to OUT. However, if *SKIP is non-zero, skip that
+ amount of data and decrease SKIP. Increment *TOTAL by the amount
+ of data written. */
+
+static int
+write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
+ wgint *written)
+{
+ if (!out)
+ return 1;
+ if (*skip > bufsize)
+ {
+ *skip -= bufsize;
+ return 1;
+ }
+ if (*skip)
+ {
+ buf += *skip;
+ bufsize -= *skip;
+ *skip = 0;
+ if (bufsize == 0)
+ return 1;
+ }
+
+ fwrite (buf, 1, bufsize, out);
+ *written += bufsize;
+
+ /* Immediately flush the downloaded data. This should not hinder
+ performance: fast downloads will arrive in large 16K chunks
+ (which stdio would write out immediately anyway), and slow
+ downloads wouldn't be limited by disk speed. */
+ fflush (out);
+ return !ferror (out);
+}
+
/* Read the contents of file descriptor FD until it the connection
terminates or a read error occurs. The data is read in portions of
up to 16K and written to OUT as it arrives. If opt.verbose is set,
the progress is shown.
TOREAD is the amount of data expected to arrive, normally only used
- by the progress gauge. However, if EXACT is set, no more than
- TOREAD octets will be read.
+ by the progress gauge.
STARTPOS is the position from which the download starts, used by
- the progress gauge. The amount of data read gets stored to
- *AMOUNT_READ. The time it took to download the data (in
- milliseconds) is stored to *ELAPSED.
+ the progress gauge. If QTYREAD is non-NULL, the value it points to
+ is incremented by the amount of data read from the network. If
+ QTYWRITTEN is non-NULL, the value it points to is incremented by
+ the amount of data written to disk. The time it took to download
+ the data (in milliseconds) is stored to ELAPSED.
The function exits and returns the amount of data read. In case of
error while reading data, -1 is returned. In case of error while
writing data, -2 is returned. */
int
-fd_read_body (int fd, FILE *out, long toread, int exact, long startpos,
- long *amount_read, double *elapsed)
+fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
+ wgint *qtyread, wgint *qtywritten, double *elapsed, int flags)
{
int ret = 0;
data arrives slowly. */
int progress_interactive = 0;
- *amount_read = 0;
+ int exact = flags & rb_read_exactly;
+ wgint skip = 0;
+
+ /* How much data we've read/written. */
+ wgint sum_read = 0;
+ wgint sum_written = 0;
+
+ if (flags & rb_skip_startpos)
+ skip = startpos;
if (opt.verbose)
{
- progress = progress_create (startpos, toread);
+ /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
+ argument to progress_create because the indicator doesn't
+ (yet) know about "skipping" data. */
+ progress = progress_create (skip ? 0 : startpos, startpos + toread);
progress_interactive = progress_interactive_p (progress);
}
means that it is unknown how much data is to arrive. However, if
EXACT is set, then toread==0 means what it says: that no data
should be read. */
- while (!exact || (*amount_read < toread))
+ while (!exact || (sum_read < toread))
{
- int rdsize = exact ? MIN (toread - *amount_read, dlbufsize) : dlbufsize;
+ int rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
double tmout = opt.read_timeout;
if (progress_interactive)
{
- double waittm;
/* For interactive progress gauges, always specify a ~1s
timeout, so that the gauge can be updated regularly even
when the data arrives very slowly or stalls. */
tmout = 0.95;
- waittm = (wtimer_read (timer) - last_successful_read_tm) / 1000;
- if (waittm + tmout > opt.read_timeout)
+ if (opt.read_timeout)
{
- /* Don't allow waiting time to exceed read timeout. */
- tmout = opt.read_timeout - waittm;
- if (tmout < 0)
+ double waittm;
+ waittm = (wtimer_read (timer) - last_successful_read_tm) / 1000;
+ if (waittm + tmout > opt.read_timeout)
{
- /* We've already exceeded the timeout. */
- ret = -1, errno = ETIMEDOUT;
- break;
+ /* Don't let total idle time exceed read timeout. */
+ tmout = opt.read_timeout - waittm;
+ if (tmout < 0)
+ {
+ /* We've already exceeded the timeout. */
+ ret = -1, errno = ETIMEDOUT;
+ break;
+ }
}
}
}
ret = fd_read (fd, dlbuf, rdsize, tmout);
- if (ret == 0 || (ret < 0 && errno != ETIMEDOUT))
+ /* when retrieving from http-proxy wget sometimes does not trust the
+ * file length reported by server.
+ * this check is to tell wget not to stubbornly try to read again and
+ * again until another errno code was received. */
+ if ( ret == -1 && errno == ETIMEDOUT && sum_read == toread && toread > 0 )
break;
+
+ if (ret == 0 || (ret < 0 && errno != ETIMEDOUT))
+ break; /* read error */
else if (ret < 0)
- ret = 0; /* timeout */
+ ret = 0; /* read timeout */
if (progress || opt.limit_rate)
{
last_successful_read_tm = wtimer_read (timer);
}
- if (ret > 0 && out != NULL)
+ if (ret > 0)
{
- fwrite (dlbuf, 1, ret, out);
- /* Immediately flush the downloaded data. This should not
- hinder performance: fast downloads will arrive in large
- 16K chunks (which stdio would write out anyway), and slow
- downloads wouldn't be limited by disk speed. */
- fflush (out);
- if (ferror (out))
+ sum_read += ret;
+ if (!write_data (out, dlbuf, ret, &skip, &sum_written))
{
ret = -2;
goto out;
if (opt.limit_rate)
limit_bandwidth (ret, timer);
- *amount_read += ret;
if (progress)
progress_update (progress, ret, wtimer_read (timer));
#ifdef WINDOWS
- if (toread > 0)
+ if (toread > 0 && !opt.quiet)
ws_percenttitle (100.0 *
- (startpos + *amount_read) / (startpos + toread));
+ (startpos + sum_read) / (startpos + toread));
#endif
}
if (ret < -1)
out:
if (progress)
progress_finish (progress, wtimer_read (timer));
+
if (elapsed)
*elapsed = wtimer_read (timer);
if (timer)
wtimer_delete (timer);
+ if (qtyread)
+ *qtyread += sum_read;
+ if (qtywritten)
+ *qtywritten += sum_written;
+
return ret;
}
\f
appropriate for the speed. If PAD is non-zero, strings will be
padded to the width of 7 characters (xxxx.xx). */
char *
-retr_rate (long bytes, double msecs, int pad)
+retr_rate (wgint bytes, double msecs, int pad)
{
static char res[20];
- static char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };
+ static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };
int units = 0;
double dlrate = calc_rate (bytes, msecs, &units);
UNITS is zero for B/s, one for KB/s, two for MB/s, and three for
GB/s. */
double
-calc_rate (long bytes, double msecs, int *units)
+calc_rate (wgint bytes, double msecs, int *units)
{
double dlrate;
xfree (url);
}
- ++global_download_count;
RESTORE_POST_DATA;
return result;
int maxlen = strlen (fname) + 1 + numdigit (opt.backups) + 1;
char *from = (char *)alloca (maxlen);
char *to = (char *)alloca (maxlen);
- struct stat sb;
+ struct_stat sb;
int i;
if (stat (fname, &sb) == 0)
projects / wget / blobdiff