X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fretr.c;h=a5198b7e591171792bf8b11360003d01c8dc4fed;hp=308289ac3378ba351a81ee5514351b666eee06c6;hb=2219d47ba301c3ea47b36291dda8eabead0fc75d;hpb=a8155e7bccfef9df2344c7aa575773efcb155ba9 diff --git a/src/retr.c b/src/retr.c index 308289ac..a5198b7e 100644 --- a/src/retr.c +++ b/src/retr.c @@ -54,6 +54,7 @@ so, delete this exception statement from your version. */ #include "connect.h" #include "hash.h" #include "convert.h" +#include "ptimer.h" #ifdef HAVE_SSL # include "gen_sslfunc.h" /* for ssl_iread */ @@ -75,7 +76,7 @@ FILE *output_stream; int output_stream_regular; static struct { - long chunk_bytes; + wgint chunk_bytes; double chunk_start; double sleep_adjust; } limit_data; @@ -92,9 +93,9 @@ limit_bandwidth_reset (void) is the timer that started at the beginning of download. */ static void -limit_bandwidth (long bytes, struct wget_timer *timer) +limit_bandwidth (wgint bytes, struct ptimer *timer) { - double delta_t = wtimer_read (timer) - limit_data.chunk_start; + double delta_t = ptimer_read (timer) - limit_data.chunk_start; double expected; limit_data.chunk_bytes += bytes; @@ -110,17 +111,18 @@ limit_bandwidth (long bytes, struct wget_timer *timer) double t0, t1; if (slp < 200) { - DEBUGP (("deferring a %.2f ms sleep (%ld/%.2f).\n", - slp, limit_data.chunk_bytes, delta_t)); + DEBUGP (("deferring a %.2f ms sleep (%s/%.2f).\n", + slp, number_to_static_string (limit_data.chunk_bytes), + delta_t)); return; } - DEBUGP (("\nsleeping %.2f ms for %ld bytes, adjust %.2f ms\n", - slp, limit_data.chunk_bytes, limit_data.sleep_adjust)); + DEBUGP (("\nsleeping %.2f ms for %s bytes, adjust %.2f ms\n", + slp, number_to_static_string (limit_data.chunk_bytes), + limit_data.sleep_adjust)); - t0 = wtimer_read (timer); + t0 = ptimer_read (timer); xsleep (slp / 1000); - wtimer_update (timer); - t1 = wtimer_read (timer); + t1 = ptimer_measure (timer); /* Due to scheduling, we probably slept slightly longer (or shorter) than desired. Calculate the difference between the @@ -130,7 +132,7 @@ limit_bandwidth (long bytes, struct wget_timer *timer) } limit_data.chunk_bytes = 0; - limit_data.chunk_start = wtimer_read (timer); + limit_data.chunk_start = ptimer_read (timer); } #ifndef MIN @@ -142,8 +144,8 @@ limit_bandwidth (long bytes, struct wget_timer *timer) of data written. */ static int -write_data (FILE *out, const char *buf, int bufsize, long *skip, - long *written) +write_data (FILE *out, const char *buf, int bufsize, wgint *skip, + wgint *written) { if (!out) return 1; @@ -192,15 +194,15 @@ write_data (FILE *out, const char *buf, int bufsize, long *skip, writing data, -2 is returned. */ int -fd_read_body (int fd, FILE *out, long toread, long startpos, - long *qtyread, long *qtywritten, double *elapsed, int flags) +fd_read_body (int fd, FILE *out, wgint toread, wgint startpos, + wgint *qtyread, wgint *qtywritten, double *elapsed, int flags) { int ret = 0; static char dlbuf[16384]; int dlbufsize = sizeof (dlbuf); - struct wget_timer *timer = NULL; + struct ptimer *timer = NULL; double last_successful_read_tm = 0; /* The progress gauge, set according to the user preferences. */ @@ -213,20 +215,21 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, int progress_interactive = 0; int exact = flags & rb_read_exactly; - long skip = 0; + wgint skip = 0; /* How much data we've read/written. */ - long sum_read = 0; - long sum_written = 0; + wgint sum_read = 0; + wgint sum_written = 0; if (flags & rb_skip_startpos) skip = startpos; if (opt.verbose) { - /* If we're skipping STARTPOS bytes, hide it from - progress_create because the indicator can't deal with it. */ - progress = progress_create (skip ? 0 : startpos, toread); + /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL + argument to progress_create because the indicator doesn't + (yet) know about "skipping" data. */ + progress = progress_create (skip ? 0 : startpos, startpos + toread); progress_interactive = progress_interactive_p (progress); } @@ -238,7 +241,7 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, the timer. */ if (progress || opt.limit_rate || elapsed) { - timer = wtimer_new (); + timer = ptimer_new (); last_successful_read_tm = 0; } @@ -259,21 +262,24 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, double tmout = opt.read_timeout; if (progress_interactive) { - double waittm; /* For interactive progress gauges, always specify a ~1s timeout, so that the gauge can be updated regularly even when the data arrives very slowly or stalls. */ tmout = 0.95; - waittm = (wtimer_read (timer) - last_successful_read_tm) / 1000; - if (waittm + tmout > opt.read_timeout) + if (opt.read_timeout) { - /* Don't let total idle time exceed read timeout. */ - tmout = opt.read_timeout - waittm; - if (tmout < 0) + double waittm; + waittm = (ptimer_read (timer) - last_successful_read_tm) / 1000; + if (waittm + tmout > opt.read_timeout) { - /* We've already exceeded the timeout. */ - ret = -1, errno = ETIMEDOUT; - break; + /* Don't let total idle time exceed read timeout. */ + tmout = opt.read_timeout - waittm; + if (tmout < 0) + { + /* We've already exceeded the timeout. */ + ret = -1, errno = ETIMEDOUT; + break; + } } } } @@ -286,9 +292,9 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, if (progress || opt.limit_rate) { - wtimer_update (timer); + ptimer_measure (timer); if (ret > 0) - last_successful_read_tm = wtimer_read (timer); + last_successful_read_tm = ptimer_read (timer); } if (ret > 0) @@ -305,9 +311,9 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, limit_bandwidth (ret, timer); if (progress) - progress_update (progress, ret, wtimer_read (timer)); + progress_update (progress, ret, ptimer_read (timer)); #ifdef WINDOWS - if (toread > 0) + if (toread > 0 && !opt.quiet) ws_percenttitle (100.0 * (startpos + sum_read) / (startpos + toread)); #endif @@ -317,12 +323,12 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, out: if (progress) - progress_finish (progress, wtimer_read (timer)); + progress_finish (progress, ptimer_read (timer)); if (elapsed) - *elapsed = wtimer_read (timer); + *elapsed = ptimer_read (timer); if (timer) - wtimer_delete (timer); + ptimer_destroy (timer); if (qtyread) *qtyread += sum_read; @@ -362,18 +368,23 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, a read. If the read returns a different amount of data, the process is retried until all data arrives safely. - BUFSIZE is the size of the initial buffer expected to read all the - data in the typical case. + SIZEHINT is the buffer size sufficient to hold all the data in the + typical case (it is used as the initial buffer size). MAXSIZE is + the maximum amount of memory this function is allowed to allocate, + or 0 if no upper limit is to be enforced. This function should be used as a building block for other functions -- see fd_read_line as a simple example. */ char * -fd_read_hunk (int fd, hunk_terminator_t hunk_terminator, int bufsize) +fd_read_hunk (int fd, hunk_terminator_t terminator, long sizehint, long maxsize) { + long bufsize = sizehint; char *hunk = xmalloc (bufsize); int tail = 0; /* tail position in HUNK */ + assert (maxsize >= bufsize); + while (1) { const char *end; @@ -387,7 +398,7 @@ fd_read_hunk (int fd, hunk_terminator_t hunk_terminator, int bufsize) xfree (hunk); return NULL; } - end = hunk_terminator (hunk, tail, pklen); + end = terminator (hunk, tail, pklen); if (end) { /* The data contains the terminator: we'll drain the data up @@ -445,7 +456,17 @@ fd_read_hunk (int fd, hunk_terminator_t hunk_terminator, int bufsize) if (tail == bufsize - 1) { + /* Double the buffer size, but refuse to allocate more than + MAXSIZE bytes. */ + if (maxsize && bufsize >= maxsize) + { + xfree (hunk); + errno = ENOMEM; + return NULL; + } bufsize <<= 1; + if (maxsize && bufsize > maxsize) + bufsize = maxsize; hunk = xrealloc (hunk, bufsize); } } @@ -461,8 +482,14 @@ line_terminator (const char *hunk, int oldlen, int peeklen) return NULL; } +/* The maximum size of the single line we agree to accept. This is + not meant to impose an arbitrary limit, but to protect the user + from Wget slurping up available memory upon encountering malicious + or buggy server output. Define it to 0 to remove the limit. */ +#define FD_READ_LINE_MAX 4096 + /* Read one line from FD and return it. The line is allocated using - malloc. + malloc, but is never larger than FD_READ_LINE_MAX. If an error occurs, or if no data can be read, NULL is returned. In the former case errno indicates the error condition, and in the @@ -471,17 +498,17 @@ line_terminator (const char *hunk, int oldlen, int peeklen) char * fd_read_line (int fd) { - return fd_read_hunk (fd, line_terminator, 128); + return fd_read_hunk (fd, line_terminator, 128, FD_READ_LINE_MAX); } /* Return a printed representation of the download rate, as appropriate for the speed. If PAD is non-zero, strings will be padded to the width of 7 characters (xxxx.xx). */ char * -retr_rate (long bytes, double msecs, int pad) +retr_rate (wgint bytes, double msecs, int pad) { static char res[20]; - static char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" }; + static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" }; int units = 0; double dlrate = calc_rate (bytes, msecs, &units); @@ -498,7 +525,7 @@ retr_rate (long bytes, double msecs, int pad) UNITS is zero for B/s, one for KB/s, two for MB/s, and three for GB/s. */ double -calc_rate (long bytes, double msecs, int *units) +calc_rate (wgint bytes, double msecs, int *units) { double dlrate; @@ -507,11 +534,12 @@ calc_rate (long bytes, double msecs, int *units) if (msecs == 0) /* If elapsed time is exactly zero, it means we're under the - granularity of the timer. This often happens on systems that - use time() for the timer. */ - msecs = wtimer_granularity (); + granularity of the timer. This can easily happen on systems + that use time() for the timer. Since the interval lies between + 0 and the timer's granularity, assume half the granularity. */ + msecs = ptimer_granularity () / 2.0; - dlrate = (double)1000 * bytes / msecs; + dlrate = 1000.0 * bytes / msecs; if (dlrate < 1024.0) *units = 0; else if (dlrate < 1024.0 * 1024.0) @@ -686,7 +714,7 @@ retrieve_url (const char *origurl, char **file, char **newloc, newloc_parsed = url_parse (mynewloc, &up_error_code); if (!newloc_parsed) { - logprintf (LOG_NOTQUIET, "%s: %s.\n", mynewloc, + logprintf (LOG_NOTQUIET, "%s: %s.\n", escnonprint_uri (mynewloc), url_error (up_error_code)); url_free (u); xfree (url); @@ -901,7 +929,7 @@ rotate_backups(const char *fname) int maxlen = strlen (fname) + 1 + numdigit (opt.backups) + 1; char *from = (char *)alloca (maxlen); char *to = (char *)alloca (maxlen); - struct stat sb; + struct_stat sb; int i; if (stat (fname, &sb) == 0)