int output_stream_regular;
\f
static struct {
- long chunk_bytes;
+ wgint chunk_bytes;
double chunk_start;
double sleep_adjust;
} limit_data;
is the timer that started at the beginning of download. */
static void
-limit_bandwidth (long bytes, struct wget_timer *timer)
+limit_bandwidth (wgint bytes, struct wget_timer *timer)
{
double delta_t = wtimer_read (timer) - limit_data.chunk_start;
double expected;
double t0, t1;
if (slp < 200)
{
- DEBUGP (("deferring a %.2f ms sleep (%ld/%.2f).\n",
- slp, limit_data.chunk_bytes, delta_t));
+ DEBUGP (("deferring a %.2f ms sleep (%s/%.2f).\n",
+ slp, number_to_static_string (limit_data.chunk_bytes),
+ delta_t));
return;
}
- DEBUGP (("\nsleeping %.2f ms for %ld bytes, adjust %.2f ms\n",
- slp, limit_data.chunk_bytes, limit_data.sleep_adjust));
+ DEBUGP (("\nsleeping %.2f ms for %s bytes, adjust %.2f ms\n",
+ slp, number_to_static_string (limit_data.chunk_bytes),
+ limit_data.sleep_adjust));
t0 = wtimer_read (timer);
xsleep (slp / 1000);
of data written. */
static int
-write_data (FILE *out, const char *buf, int bufsize, long *skip,
- long *written)
+write_data (FILE *out, const char *buf, int bufsize, wgint *skip,
+ wgint *written)
{
if (!out)
return 1;
writing data, -2 is returned. */
int
-fd_read_body (int fd, FILE *out, long toread, long startpos,
- long *qtyread, long *qtywritten, double *elapsed, int flags)
+fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
+ wgint *qtyread, wgint *qtywritten, double *elapsed, int flags)
{
int ret = 0;
int progress_interactive = 0;
int exact = flags & rb_read_exactly;
- long skip = 0;
+ wgint skip = 0;
/* How much data we've read/written. */
- long sum_read = 0;
- long sum_written = 0;
+ wgint sum_read = 0;
+ wgint sum_written = 0;
if (flags & rb_skip_startpos)
skip = startpos;
}
ret = fd_read (fd, dlbuf, rdsize, tmout);
+ /* when retrieving from http-proxy wget sometimes does not trust the
+ * file length reported by server.
+ * this check is to tell wget not to stubbornly try to read again and
+ * again until another errno code was received. */
+ if ( ret == -1 && errno == ETIMEDOUT && sum_read == toread && toread > 0 )
+ break;
+
if (ret == 0 || (ret < 0 && errno != ETIMEDOUT))
break; /* read error */
else if (ret < 0)
if (progress)
progress_update (progress, ret, wtimer_read (timer));
#ifdef WINDOWS
- if (toread > 0)
+ if (toread > 0 && !opt.quiet)
ws_percenttitle (100.0 *
(startpos + sum_read) / (startpos + toread));
#endif
a read. If the read returns a different amount of data, the
process is retried until all data arrives safely.
- BUFSIZE is the size of the initial buffer expected to read all the
- data in the typical case.
+ SIZEHINT is the buffer size sufficient to hold all the data in the
+ typical case (it is used as the initial buffer size). MAXSIZE is
+ the maximum amount of memory this function is allowed to allocate,
+ or 0 if no upper limit is to be enforced.
This function should be used as a building block for other
functions -- see fd_read_line as a simple example. */
char *
-fd_read_hunk (int fd, hunk_terminator_t hunk_terminator, int bufsize)
+fd_read_hunk (int fd, hunk_terminator_t terminator, long sizehint, long maxsize)
{
+ long bufsize = sizehint;
char *hunk = xmalloc (bufsize);
int tail = 0; /* tail position in HUNK */
+ assert (maxsize >= bufsize);
+
while (1)
{
const char *end;
xfree (hunk);
return NULL;
}
- end = hunk_terminator (hunk, tail, pklen);
+ end = terminator (hunk, tail, pklen);
if (end)
{
/* The data contains the terminator: we'll drain the data up
if (tail == bufsize - 1)
{
+ /* Double the buffer size, but refuse to allocate more than
+ MAXSIZE bytes. */
+ if (maxsize && bufsize >= maxsize)
+ {
+ xfree (hunk);
+ errno = ENOMEM;
+ return NULL;
+ }
bufsize <<= 1;
+ if (maxsize && bufsize > maxsize)
+ bufsize = maxsize;
hunk = xrealloc (hunk, bufsize);
}
}
return NULL;
}
+/* The maximum size of the single line we agree to accept. This is
+ not meant to impose an arbitrary limit, but to protect the user
+ from Wget slurping up available memory upon encountering malicious
+ or buggy server output. Define it to 0 to remove the limit. */
+#define FD_READ_LINE_MAX 4096
+
/* Read one line from FD and return it. The line is allocated using
- malloc.
+ malloc, but is never larger than FD_READ_LINE_MAX.
If an error occurs, or if no data can be read, NULL is returned.
In the former case errno indicates the error condition, and in the
char *
fd_read_line (int fd)
{
- return fd_read_hunk (fd, line_terminator, 128);
+ return fd_read_hunk (fd, line_terminator, 128, FD_READ_LINE_MAX);
}
\f
/* Return a printed representation of the download rate, as
appropriate for the speed. If PAD is non-zero, strings will be
padded to the width of 7 characters (xxxx.xx). */
char *
-retr_rate (long bytes, double msecs, int pad)
+retr_rate (wgint bytes, double msecs, int pad)
{
static char res[20];
- static char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };
+ static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" };
int units = 0;
double dlrate = calc_rate (bytes, msecs, &units);
UNITS is zero for B/s, one for KB/s, two for MB/s, and three for
GB/s. */
double
-calc_rate (long bytes, double msecs, int *units)
+calc_rate (wgint bytes, double msecs, int *units)
{
double dlrate;
newloc_parsed = url_parse (mynewloc, &up_error_code);
if (!newloc_parsed)
{
- logprintf (LOG_NOTQUIET, "%s: %s.\n", mynewloc,
+ logprintf (LOG_NOTQUIET, "%s: %s.\n", escnonprint_uri (mynewloc),
url_error (up_error_code));
url_free (u);
xfree (url);
int maxlen = strlen (fname) + 1 + numdigit (opt.backups) + 1;
char *from = (char *)alloca (maxlen);
char *to = (char *)alloca (maxlen);
- struct stat sb;
+ struct_stat sb;
int i;
if (stat (fname, &sb) == 0)
projects / wget / blobdiff