X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Fretr.c;h=c5c1532ba7188c7c39ef0dd6315e3fdf984a76e9;hp=6ea558fa49a36eb79e01695a114777df4c46169c;hb=0967c21094580317353f0742c4836c5bbea34059;hpb=9179f2e699cd0fbd5678e4b42a3072f1c2afa30c diff --git a/src/retr.c b/src/retr.c index 6ea558fa..c5c1532b 100644 --- a/src/retr.c +++ b/src/retr.c @@ -54,10 +54,7 @@ so, delete this exception statement from your version. */ #include "connect.h" #include "hash.h" #include "convert.h" - -#ifdef HAVE_SSL -# include "gen_sslfunc.h" /* for ssl_iread */ -#endif +#include "ptimer.h" #ifndef errno extern int errno; @@ -75,7 +72,7 @@ FILE *output_stream; int output_stream_regular; static struct { - long chunk_bytes; + wgint chunk_bytes; double chunk_start; double sleep_adjust; } limit_data; @@ -92,9 +89,9 @@ limit_bandwidth_reset (void) is the timer that started at the beginning of download. */ static void -limit_bandwidth (long bytes, struct wget_timer *timer) +limit_bandwidth (wgint bytes, struct ptimer *timer) { - double delta_t = wtimer_read (timer) - limit_data.chunk_start; + double delta_t = ptimer_read (timer) - limit_data.chunk_start; double expected; limit_data.chunk_bytes += bytes; @@ -110,17 +107,18 @@ limit_bandwidth (long bytes, struct wget_timer *timer) double t0, t1; if (slp < 200) { - DEBUGP (("deferring a %.2f ms sleep (%ld/%.2f).\n", - slp, limit_data.chunk_bytes, delta_t)); + DEBUGP (("deferring a %.2f ms sleep (%s/%.2f).\n", + slp, number_to_static_string (limit_data.chunk_bytes), + delta_t)); return; } - DEBUGP (("\nsleeping %.2f ms for %ld bytes, adjust %.2f ms\n", - slp, limit_data.chunk_bytes, limit_data.sleep_adjust)); + DEBUGP (("\nsleeping %.2f ms for %s bytes, adjust %.2f ms\n", + slp, number_to_static_string (limit_data.chunk_bytes), + limit_data.sleep_adjust)); - t0 = wtimer_read (timer); + t0 = ptimer_read (timer); xsleep (slp / 1000); - wtimer_update (timer); - t1 = wtimer_read (timer); + t1 = ptimer_measure (timer); /* Due to scheduling, we probably slept slightly longer (or shorter) than desired. Calculate the difference between the @@ -130,7 +128,7 @@ limit_bandwidth (long bytes, struct wget_timer *timer) } limit_data.chunk_bytes = 0; - limit_data.chunk_start = wtimer_read (timer); + limit_data.chunk_start = ptimer_read (timer); } #ifndef MIN @@ -142,8 +140,8 @@ limit_bandwidth (long bytes, struct wget_timer *timer) of data written. */ static int -write_data (FILE *out, const char *buf, int bufsize, long *skip, - long *written) +write_data (FILE *out, const char *buf, int bufsize, wgint *skip, + wgint *written) { if (!out) return 1; @@ -192,15 +190,15 @@ write_data (FILE *out, const char *buf, int bufsize, long *skip, writing data, -2 is returned. */ int -fd_read_body (int fd, FILE *out, long toread, long startpos, - long *qtyread, long *qtywritten, double *elapsed, int flags) +fd_read_body (int fd, FILE *out, wgint toread, wgint startpos, + wgint *qtyread, wgint *qtywritten, double *elapsed, int flags) { int ret = 0; static char dlbuf[16384]; int dlbufsize = sizeof (dlbuf); - struct wget_timer *timer = NULL; + struct ptimer *timer = NULL; double last_successful_read_tm = 0; /* The progress gauge, set according to the user preferences. */ @@ -213,20 +211,21 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, int progress_interactive = 0; int exact = flags & rb_read_exactly; - long skip = 0; + wgint skip = 0; /* How much data we've read/written. */ - long sum_read = 0; - long sum_written = 0; + wgint sum_read = 0; + wgint sum_written = 0; if (flags & rb_skip_startpos) skip = startpos; if (opt.verbose) { - /* If we're skipping STARTPOS bytes, hide it from - progress_create because the indicator can't deal with it. */ - progress = progress_create (skip ? 0 : startpos, toread); + /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL + argument to progress_create because the indicator doesn't + (yet) know about "skipping" data. */ + progress = progress_create (skip ? 0 : startpos, startpos + toread); progress_interactive = progress_interactive_p (progress); } @@ -238,7 +237,7 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, the timer. */ if (progress || opt.limit_rate || elapsed) { - timer = wtimer_new (); + timer = ptimer_new (); last_successful_read_tm = 0; } @@ -266,7 +265,7 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, if (opt.read_timeout) { double waittm; - waittm = (wtimer_read (timer) - last_successful_read_tm) / 1000; + waittm = (ptimer_read (timer) - last_successful_read_tm) / 1000; if (waittm + tmout > opt.read_timeout) { /* Don't let total idle time exceed read timeout. */ @@ -282,16 +281,16 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, } ret = fd_read (fd, dlbuf, rdsize, tmout); - if (ret == 0 || (ret < 0 && errno != ETIMEDOUT)) - break; /* read error */ - else if (ret < 0) - ret = 0; /* read timeout */ + if (progress_interactive && ret < 0 && errno == ETIMEDOUT) + ret = 0; /* interactive timeout, handled above */ + else if (ret <= 0) + break; /* EOF or read error */ if (progress || opt.limit_rate) { - wtimer_update (timer); + ptimer_measure (timer); if (ret > 0) - last_successful_read_tm = wtimer_read (timer); + last_successful_read_tm = ptimer_read (timer); } if (ret > 0) @@ -300,7 +299,7 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, if (!write_data (out, dlbuf, ret, &skip, &sum_written)) { ret = -2; - goto out; + goto out_; } } @@ -308,9 +307,9 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, limit_bandwidth (ret, timer); if (progress) - progress_update (progress, ret, wtimer_read (timer)); + progress_update (progress, ret, ptimer_read (timer)); #ifdef WINDOWS - if (toread > 0) + if (toread > 0 && !opt.quiet) ws_percenttitle (100.0 * (startpos + sum_read) / (startpos + toread)); #endif @@ -318,14 +317,14 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, if (ret < -1) ret = -1; - out: + out_: if (progress) - progress_finish (progress, wtimer_read (timer)); + progress_finish (progress, ptimer_read (timer)); if (elapsed) - *elapsed = wtimer_read (timer); + *elapsed = ptimer_read (timer); if (timer) - wtimer_delete (timer); + ptimer_destroy (timer); if (qtyread) *qtyread += sum_read; @@ -365,18 +364,23 @@ fd_read_body (int fd, FILE *out, long toread, long startpos, a read. If the read returns a different amount of data, the process is retried until all data arrives safely. - BUFSIZE is the size of the initial buffer expected to read all the - data in the typical case. + SIZEHINT is the buffer size sufficient to hold all the data in the + typical case (it is used as the initial buffer size). MAXSIZE is + the maximum amount of memory this function is allowed to allocate, + or 0 if no upper limit is to be enforced. This function should be used as a building block for other functions -- see fd_read_line as a simple example. */ char * -fd_read_hunk (int fd, hunk_terminator_t hunk_terminator, int bufsize) +fd_read_hunk (int fd, hunk_terminator_t terminator, long sizehint, long maxsize) { + long bufsize = sizehint; char *hunk = xmalloc (bufsize); int tail = 0; /* tail position in HUNK */ + assert (maxsize >= bufsize); + while (1) { const char *end; @@ -384,13 +388,13 @@ fd_read_hunk (int fd, hunk_terminator_t hunk_terminator, int bufsize) /* First, peek at the available data. */ - pklen = fd_peek (fd, hunk + tail, bufsize - 1 - tail, -1); + pklen = fd_peek (fd, hunk + tail, bufsize - 1 - tail, -1.0); if (pklen < 0) { xfree (hunk); return NULL; } - end = hunk_terminator (hunk, tail, pklen); + end = terminator (hunk, tail, pklen); if (end) { /* The data contains the terminator: we'll drain the data up @@ -417,7 +421,7 @@ fd_read_hunk (int fd, hunk_terminator_t hunk_terminator, int bufsize) how much data we'll get. (Some TCP stacks are notorious for read returning less data than the previous MSG_PEEK.) */ - rdlen = fd_read (fd, hunk + tail, remain, 0); + rdlen = fd_read (fd, hunk + tail, remain, 0.0); if (rdlen < 0) { xfree_null (hunk); @@ -448,7 +452,17 @@ fd_read_hunk (int fd, hunk_terminator_t hunk_terminator, int bufsize) if (tail == bufsize - 1) { + /* Double the buffer size, but refuse to allocate more than + MAXSIZE bytes. */ + if (maxsize && bufsize >= maxsize) + { + xfree (hunk); + errno = ENOMEM; + return NULL; + } bufsize <<= 1; + if (maxsize && bufsize > maxsize) + bufsize = maxsize; hunk = xrealloc (hunk, bufsize); } } @@ -464,8 +478,14 @@ line_terminator (const char *hunk, int oldlen, int peeklen) return NULL; } +/* The maximum size of the single line we agree to accept. This is + not meant to impose an arbitrary limit, but to protect the user + from Wget slurping up available memory upon encountering malicious + or buggy server output. Define it to 0 to remove the limit. */ +#define FD_READ_LINE_MAX 4096 + /* Read one line from FD and return it. The line is allocated using - malloc. + malloc, but is never larger than FD_READ_LINE_MAX. If an error occurs, or if no data can be read, NULL is returned. In the former case errno indicates the error condition, and in the @@ -474,17 +494,17 @@ line_terminator (const char *hunk, int oldlen, int peeklen) char * fd_read_line (int fd) { - return fd_read_hunk (fd, line_terminator, 128); + return fd_read_hunk (fd, line_terminator, 128, FD_READ_LINE_MAX); } /* Return a printed representation of the download rate, as appropriate for the speed. If PAD is non-zero, strings will be padded to the width of 7 characters (xxxx.xx). */ char * -retr_rate (long bytes, double msecs, int pad) +retr_rate (wgint bytes, double msecs, int pad) { static char res[20]; - static char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" }; + static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" }; int units = 0; double dlrate = calc_rate (bytes, msecs, &units); @@ -501,7 +521,7 @@ retr_rate (long bytes, double msecs, int pad) UNITS is zero for B/s, one for KB/s, two for MB/s, and three for GB/s. */ double -calc_rate (long bytes, double msecs, int *units) +calc_rate (wgint bytes, double msecs, int *units) { double dlrate; @@ -510,11 +530,12 @@ calc_rate (long bytes, double msecs, int *units) if (msecs == 0) /* If elapsed time is exactly zero, it means we're under the - granularity of the timer. This often happens on systems that - use time() for the timer. */ - msecs = wtimer_granularity (); + resolution of the timer. This can easily happen on systems + that use time() for the timer. Since the interval lies between + 0 and the timer's resolution, assume half the resolution. */ + msecs = ptimer_resolution () / 2.0; - dlrate = (double)1000 * bytes / msecs; + dlrate = 1000.0 * bytes / msecs; if (dlrate < 1024.0) *units = 0; else if (dlrate < 1024.0 * 1024.0) @@ -689,7 +710,7 @@ retrieve_url (const char *origurl, char **file, char **newloc, newloc_parsed = url_parse (mynewloc, &up_error_code); if (!newloc_parsed) { - logprintf (LOG_NOTQUIET, "%s: %s.\n", mynewloc, + logprintf (LOG_NOTQUIET, "%s: %s.\n", escnonprint_uri (mynewloc), url_error (up_error_code)); url_free (u); xfree (url); @@ -809,8 +830,8 @@ retrieve_from_file (const char *file, int html, int *count) if (filename && opt.delete_after && file_exists_p (filename)) { - DEBUGP (("Removing file due to --delete-after in" - " retrieve_from_file():\n")); + DEBUGP (("\ +Removing file due to --delete-after in retrieve_from_file():\n")); logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename); if (unlink (filename)) logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); @@ -858,7 +879,7 @@ sleep_between_retrievals (int count) /* If opt.waitretry is specified and this is a retry, wait for COUNT-1 number of seconds, or for opt.waitretry seconds. */ if (count <= opt.waitretry) - xsleep (count - 1); + xsleep (count - 1.0); else xsleep (opt.waitretry); } @@ -904,7 +925,7 @@ rotate_backups(const char *fname) int maxlen = strlen (fname) + 1 + numdigit (opt.backups) + 1; char *from = (char *)alloca (maxlen); char *to = (char *)alloca (maxlen); - struct stat sb; + struct_stat sb; int i; if (stat (fname, &sb) == 0) @@ -971,7 +992,7 @@ getproxy (struct url *u) } /* Should a host be accessed through proxy, concerning no_proxy? */ -int +static int no_proxy_match (const char *host, const char **no_proxy) { if (!no_proxy)