From: hniksic Date: Mon, 15 Sep 2003 21:14:15 +0000 (-0700) Subject: [svn] Update progress code to use higher timer resolution. X-Git-Tag: v1.13~1722 X-Git-Url: http://sjero.net/git/?p=wget;a=commitdiff_plain;h=9228f0bf53d3b42459daeb28372196a007de3014 [svn] Update progress code to use higher timer resolution. Message-ID: --- diff --git a/src/ChangeLog b/src/ChangeLog index 503f2759..b3a321fe 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,11 @@ +2003-09-15 Hrvoje Niksic + + * progress.c (update_speed_ring): Moved the speed ring update to a + separate function and documented it better. + + * progress.c: Use `double' for most timers to support granularity + smaller than 1ms. + 2003-09-15 Hrvoje Niksic * wget.h (XDIGIT_TO_XCHAR): Implement as index into a literal diff --git a/src/ftp.c b/src/ftp.c index d70969ad..966b90b5 100644 --- a/src/ftp.c +++ b/src/ftp.c @@ -69,7 +69,7 @@ typedef struct int st; /* connection status */ int cmd; /* command code */ struct rbuf rbuf; /* control connection buffer */ - long dltime; /* time of the download */ + double dltime; /* time of the download in msecs */ enum stype rs; /* remote system reported by ftp server */ char *id; /* initial directory */ char *target; /* target file name */ diff --git a/src/http.c b/src/http.c index 82c6d8de..abaa4bdb 100644 --- a/src/http.c +++ b/src/http.c @@ -584,7 +584,7 @@ struct http_stat char *remote_time; /* remote time-stamp string */ char *error; /* textual HTTP error */ int statcode; /* status code */ - long dltime; /* time of the download */ + double dltime; /* time of the download in msecs */ int no_truncate; /* whether truncating the file is forbidden. */ const char *referer; /* value of the referer header. */ diff --git a/src/progress.c b/src/progress.c index 0288355d..0e626796 100644 --- a/src/progress.c +++ b/src/progress.c @@ -52,21 +52,21 @@ so, delete this exception statement from your version. */ struct progress_implementation { char *name; void *(*create) PARAMS ((long, long)); - void (*update) PARAMS ((void *, long, long)); - void (*finish) PARAMS ((void *, long)); + void (*update) PARAMS ((void *, long, double)); + void (*finish) PARAMS ((void *, double)); void (*set_params) PARAMS ((const char *)); }; /* Necessary forward declarations. */ static void *dot_create PARAMS ((long, long)); -static void dot_update PARAMS ((void *, long, long)); -static void dot_finish PARAMS ((void *, long)); +static void dot_update PARAMS ((void *, long, double)); +static void dot_finish PARAMS ((void *, double)); static void dot_set_params PARAMS ((const char *)); static void *bar_create PARAMS ((long, long)); -static void bar_update PARAMS ((void *, long, long)); -static void bar_finish PARAMS ((void *, long)); +static void bar_update PARAMS ((void *, long, double)); +static void bar_finish PARAMS ((void *, double)); static void bar_set_params PARAMS ((const char *)); static struct progress_implementation implementations[] = { @@ -172,7 +172,7 @@ progress_create (long initial, long total) time in milliseconds since the beginning of the download. */ void -progress_update (void *progress, long howmuch, long dltime) +progress_update (void *progress, long howmuch, double dltime) { current_impl->update (progress, howmuch, dltime); } @@ -181,7 +181,7 @@ progress_update (void *progress, long howmuch, long dltime) PROGRESS object, the further use of which is not allowed. */ void -progress_finish (void *progress, long dltime) +progress_finish (void *progress, double dltime) { current_impl->finish (progress, dltime); } @@ -198,7 +198,7 @@ struct dot_progress { int rows; /* number of rows printed so far */ int dots; /* number of dots printed in this row */ - long last_timer_value; + double last_timer_value; }; /* Dot-progress backend for progress_create. */ @@ -260,7 +260,7 @@ print_percentage (long bytes, long expected) } static void -print_download_speed (struct dot_progress *dp, long bytes, long dltime) +print_download_speed (struct dot_progress *dp, long bytes, double dltime) { logprintf (LOG_VERBOSE, " %s", retr_rate (bytes, dltime - dp->last_timer_value, 1)); @@ -270,7 +270,7 @@ print_download_speed (struct dot_progress *dp, long bytes, long dltime) /* Dot-progress backend for progress_update. */ static void -dot_update (void *progress, long howmuch, long dltime) +dot_update (void *progress, long howmuch, double dltime) { struct dot_progress *dp = progress; int dot_bytes = opt.dot_bytes; @@ -310,7 +310,7 @@ dot_update (void *progress, long howmuch, long dltime) /* Dot-progress backend for progress_finish. */ static void -dot_finish (void *progress, long dltime) +dot_finish (void *progress, double dltime) { struct dot_progress *dp = progress; int dot_bytes = opt.dot_bytes; @@ -413,13 +413,14 @@ dot_set_params (const char *params) static int screen_width = DEFAULT_SCREEN_WIDTH; -/* Size of the history table for download speeds. */ +/* Size of the download speed history ring. */ #define DLSPEED_HISTORY_SIZE 30 -/* The time interval in milliseconds below which we increase old - history entries rather than overwriting them. That interval - represents the scope of the download speed history. */ -#define DLSPEED_HISTORY_MAX_INTERVAL 3000 +/* The minimum time length of a history sample. By default, each + sample is at least 100ms long, which means that, over the course of + 30 samples, "current" download speed spans at least 3s into the + past. */ +#define DLSPEED_SAMPLE_MIN 100 struct bar_progress { long initial_length; /* how many bytes have been downloaded @@ -428,7 +429,9 @@ struct bar_progress { download finishes */ long count; /* bytes downloaded so far */ - long last_screen_update; /* time of the last screen update. */ + double last_screen_update; /* time of the last screen update, + measured since the beginning of + download. */ int width; /* screen width we're using at the time the progress gauge was @@ -449,19 +452,26 @@ struct bar_progress { int pos; long times[DLSPEED_HISTORY_SIZE]; long bytes[DLSPEED_HISTORY_SIZE]; - long summed_times; - long summed_bytes; - long previous_time; + + /* The sum of times and bytes respectively, maintained for + efficiency. */ + long total_time; + long total_bytes; } hist; + double recent_start; /* timestamp of beginning of current + position. */ + long recent_bytes; /* bytes downloaded so far. */ + /* create_image() uses these to make sure that ETA information doesn't flash. */ - long last_eta_time; /* time of the last update to download - speed and ETA. */ + double last_eta_time; /* time of the last update to download + speed and ETA, measured since the + beginning of download. */ long last_eta_value; }; -static void create_image PARAMS ((struct bar_progress *, long)); +static void create_image PARAMS ((struct bar_progress *, double)); static void display_image PARAMS ((char *)); static void * @@ -492,13 +502,13 @@ bar_create (long initial, long total) return bp; } +static void update_speed_ring PARAMS ((struct bar_progress *, long, double)); + static void -bar_update (void *progress, long howmuch, long dltime) +bar_update (void *progress, long howmuch, double dltime) { struct bar_progress *bp = progress; - struct bar_progress_hist *hist = &bp->hist; int force_screen_update = 0; - long delta_time = dltime - hist->previous_time; bp->count += howmuch; if (bp->total_length > 0 @@ -510,54 +520,7 @@ bar_update (void *progress, long howmuch, long dltime) equal to the expected size doesn't abort. */ bp->total_length = bp->initial_length + bp->count; - /* This code attempts to determine the current download speed. We - measure the speed over the interval of approximately three - seconds, in subintervals no smaller than 0.1s. In other words, - we maintain and use the history of 30 most recent reads, where a - "read" consists of one or more network reads, up until the point - where a subinterval is filled. */ - - if (hist->times[hist->pos] - >= DLSPEED_HISTORY_MAX_INTERVAL / DLSPEED_HISTORY_SIZE) - { - /* The subinterval at POS has been used up. Move on to the next - position. */ - if (++hist->pos == DLSPEED_HISTORY_SIZE) - hist->pos = 0; - - /* Invalidate old data (from the previous cycle) at this - position. */ - hist->summed_times -= hist->times[hist->pos]; - hist->summed_bytes -= hist->bytes[hist->pos]; - hist->times[hist->pos] = delta_time; - hist->bytes[hist->pos] = howmuch; - } - else - { - /* Increment the data at POS. */ - hist->times[hist->pos] += delta_time; - hist->bytes[hist->pos] += howmuch; - } - - hist->summed_times += delta_time; - hist->summed_bytes += howmuch; - hist->previous_time = dltime; - -#if 0 - /* Sledgehammer check that summed_times and summed_bytes are - accurate. */ - { - int i; - long sumt = 0, sumb = 0; - for (i = 0; i < DLSPEED_HISTORY_SIZE; i++) - { - sumt += hist->times[i]; - sumb += hist->bytes[i]; - } - assert (sumt == hist->summed_times); - assert (sumb == hist->summed_bytes); - } -#endif + update_speed_ring (bp, howmuch, dltime); if (screen_width - 1 != bp->width) { @@ -576,7 +539,7 @@ bar_update (void *progress, long howmuch, long dltime) } static void -bar_finish (void *progress, long dltime) +bar_finish (void *progress, double dltime) { struct bar_progress *bp = progress; @@ -594,6 +557,77 @@ bar_finish (void *progress, long dltime) xfree (bp); } +/* This code attempts to maintain the notion of a "current" download + speed, over the course of no less than 3s. (Shorter intervals + produce very erratic results.) + + To do so, it samples the speed in 0.1s intervals and stores the + recorded samples in a FIFO history ring. The ring stores no more + than 30 intervals, hence the history covers the period of at least + three seconds and at most 30 reads into the past. This method + should produce good results for both very fast and very slow + downloads. + + The idea is that for fast downloads, we get the speed over exactly + the last three seconds. For slow downloads (where a network read + takes more than 0.1s to complete), we get the speed over a larger + time period, as large as it takes to complete thirty reads. This + is good because slow downloads tend to fluctuate more and a + 3-second average would be very erratic. */ + +static void +update_speed_ring (struct bar_progress *bp, long howmuch, double dltime) +{ + struct bar_progress_hist *hist = &bp->hist; + double recent_age = dltime - bp->recent_start; + + /* Update the download count. */ + bp->recent_bytes += howmuch; + + /* For very small time intervals, we return after having updated the + "recent" download count. When its age reaches or exceeds minimum + sample time, it will be recorded in the history ring. */ + if (recent_age < DLSPEED_SAMPLE_MIN) + return; + + /* Store "recent" bytes and download time to history ring at the + position POS. */ + + /* To correctly maintain the totals, first invalidate existing data + (least recent in time) at this position. */ + hist->total_time -= hist->times[hist->pos]; + hist->total_bytes -= hist->bytes[hist->pos]; + + /* Now store the new data and update the totals. */ + hist->times[hist->pos] = recent_age; + hist->bytes[hist->pos] = bp->recent_bytes; + hist->total_time += recent_age; + hist->total_bytes += bp->recent_bytes; + + /* Start a new "recent" period. */ + bp->recent_start = dltime; + bp->recent_bytes = 0; + + /* Advance the current ring position. */ + if (++hist->pos == DLSPEED_HISTORY_SIZE) + hist->pos = 0; + +#if 0 + /* Sledgehammer check to verify that the totals are accurate. */ + { + int i; + double sumt = 0, sumb = 0; + for (i = 0; i < DLSPEED_HISTORY_SIZE; i++) + { + sumt += hist->times[i]; + sumb += hist->bytes[i]; + } + assert (sumt == hist->total_time); + assert (sumb == hist->total_bytes); + } +#endif +} + #define APPEND_LITERAL(s) do { \ memcpy (p, s, sizeof (s) - 1); \ p += sizeof (s) - 1; \ @@ -604,7 +638,7 @@ bar_finish (void *progress, long dltime) #endif static void -create_image (struct bar_progress *bp, long dl_total_time) +create_image (struct bar_progress *bp, double dl_total_time) { char *p = bp->buffer; long size = bp->initial_length + bp->count; @@ -711,12 +745,13 @@ create_image (struct bar_progress *bp, long dl_total_time) p += strlen (p); /* " 1012.45K/s" */ - if (hist->summed_times && hist->summed_bytes) + if (hist->total_time && hist->total_bytes) { static char *short_units[] = { "B/s", "K/s", "M/s", "G/s" }; int units = 0; - double dlrate; - dlrate = calc_rate (hist->summed_bytes, hist->summed_times, &units); + long bytes = hist->total_bytes + bp->recent_bytes; + double tm = hist->total_time + dl_total_time - bp->recent_start; + double dlrate = calc_rate (bytes, tm, &units); sprintf (p, " %7.2f%s", dlrate, short_units[units]); p += strlen (p); } @@ -738,11 +773,11 @@ create_image (struct bar_progress *bp, long dl_total_time) else { /* Calculate ETA using the average download speed to predict - the future speed. If you want to use the current speed - instead, replace dl_total_time with hist->summed_times - and bp->count with hist->summed_bytes. I found that - doing that results in a very jerky and ultimately - unreliable ETA. */ + the future speed. If you want to use a speed averaged + over a more recent period, replace dl_total_time with + hist->total_time and bp->count with hist->total_bytes. + I found that doing that results in a very jerky and + ultimately unreliable ETA. */ double time_sofar = (double)dl_total_time / 1000; long bytes_remaining = bp->total_length - size; eta = (long) (time_sofar * bytes_remaining / bp->count); diff --git a/src/progress.h b/src/progress.h index 28174ecd..dad06f59 100644 --- a/src/progress.h +++ b/src/progress.h @@ -35,8 +35,8 @@ void set_progress_implementation PARAMS ((const char *)); void progress_schedule_redirect PARAMS ((void)); void *progress_create PARAMS ((long, long)); -void progress_update PARAMS ((void *, long, long)); -void progress_finish PARAMS ((void *, long)); +void progress_update PARAMS ((void *, long, double)); +void progress_finish PARAMS ((void *, double)); RETSIGTYPE progress_handle_sigwinch PARAMS ((int)); diff --git a/src/retr.c b/src/retr.c index 50b209da..e619f1cb 100644 --- a/src/retr.c +++ b/src/retr.c @@ -5,8 +5,8 @@ This file is part of GNU Wget. GNU Wget is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. +the Free Software Foundation; either version 2 of the License, or (at +your option) any later version. GNU Wget is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -68,7 +68,7 @@ int global_download_count; static struct { long bytes; - long dltime; + double dltime; } limit_data; static void @@ -84,26 +84,26 @@ limit_bandwidth_reset (void) TIMER the timer, and ADJUSTMENT the previous. */ static void -limit_bandwidth (long bytes, long delta) +limit_bandwidth (long bytes, double delta) { - long expected; + double expected; limit_data.bytes += bytes; limit_data.dltime += delta; - expected = (long)(1000.0 * limit_data.bytes / opt.limit_rate); + expected = 1000.0 * limit_data.bytes / opt.limit_rate; if (expected > limit_data.dltime) { - long slp = expected - limit_data.dltime; + double slp = expected - limit_data.dltime; if (slp < 200) { - DEBUGP (("deferring a %ld ms sleep (%ld/%ld) until later.\n", + DEBUGP (("deferring a %.2f ms sleep (%ld/%.2f).\n", slp, limit_data.bytes, limit_data.dltime)); return; } - DEBUGP (("sleeping %ld ms\n", slp)); - usleep (1000 * slp); + DEBUGP (("sleeping %.2f ms\n", slp)); + usleep ((unsigned long) (1000 * slp)); } limit_data.bytes = 0; @@ -135,13 +135,13 @@ limit_bandwidth (long bytes, long delta) from fd immediately, flush or discard the buffer. */ int get_contents (int fd, FILE *fp, long *len, long restval, long expected, - struct rbuf *rbuf, int use_expected, long *elapsed) + struct rbuf *rbuf, int use_expected, double *elapsed) { int res = 0; - static char c[8192]; + static char c[16384]; void *progress = NULL; struct wget_timer *timer = wtimer_allocate (); - long dltime = 0, last_dltime = 0; + double dltime = 0, last_dltime = 0; *len = restval; @@ -236,7 +236,7 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected, appropriate for the speed. If PAD is non-zero, strings will be padded to the width of 7 characters (xxxx.xx). */ char * -retr_rate (long bytes, long msecs, int pad) +retr_rate (long bytes, double msecs, int pad) { static char res[20]; static char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" }; @@ -256,7 +256,7 @@ retr_rate (long bytes, long msecs, int pad) UNITS is zero for B/s, one for KB/s, two for MB/s, and three for GB/s. */ double -calc_rate (long bytes, long msecs, int *units) +calc_rate (long bytes, double msecs, int *units) { double dlrate; @@ -264,9 +264,9 @@ calc_rate (long bytes, long msecs, int *units) assert (bytes >= 0); if (msecs == 0) - /* If elapsed time is 0, it means we're under the granularity of - the timer. This often happens on systems that use time() for - the timer. */ + /* If elapsed time is exactly zero, it means we're under the + granularity of the timer. This often happens on systems that + use time() for the timer. */ msecs = wtimer_granularity (); dlrate = (double)1000 * bytes / msecs; diff --git a/src/retr.h b/src/retr.h index 0ceb0eab..c2867c41 100644 --- a/src/retr.h +++ b/src/retr.h @@ -33,14 +33,14 @@ so, delete this exception statement from your version. */ #include "rbuf.h" int get_contents PARAMS ((int, FILE *, long *, long, long, struct rbuf *, - int, long *)); + int, double *)); uerr_t retrieve_url PARAMS ((const char *, char **, char **, const char *, int *)); uerr_t retrieve_from_file PARAMS ((const char *, int, int *)); -char *retr_rate PARAMS ((long, long, int)); -double calc_rate PARAMS ((long, long, int *)); +char *retr_rate PARAMS ((long, double, int)); +double calc_rate PARAMS ((long, double, int *)); void printwhat PARAMS ((int, int)); void downloaded_increase PARAMS ((unsigned long)); diff --git a/src/utils.c b/src/utils.c index 72b95279..74e4552f 100644 --- a/src/utils.c +++ b/src/utils.c @@ -1602,9 +1602,20 @@ wtimer_sys_set (wget_sys_time *wst) #endif #ifdef TIMER_WINDOWS + /* We use GetSystemTime to get the elapsed time. MSDN warns that + system clock adjustments can skew the output of GetSystemTime + when used as a timer and gives preference to GetTickCount and + high-resolution timers. But GetTickCount can overflow, and hires + timers are typically used for profiling, not for regular time + measurement. Since we handle clock skew anyway, we just use + GetSystemTime. */ FILETIME ft; SYSTEMTIME st; GetSystemTime (&st); + + /* As recommended by MSDN, we convert SYSTEMTIME to FILETIME, copy + FILETIME to ULARGE_INTEGER, and use regular 64-bit integer + arithmetic on that. */ SystemTimeToFileTime (&st, &ft); wst->HighPart = ft.dwHighDateTime; wst->LowPart = ft.dwLowDateTime; @@ -1643,7 +1654,8 @@ wtimer_sys_diff (wget_sys_time *wst1, wget_sys_time *wst2) /* Return the number of milliseconds elapsed since the timer was last reset. It is allowed to call this function more than once to get - increasingly higher elapsed values. */ + increasingly higher elapsed values. These timers handle clock + skew. */ double wtimer_elapsed (struct wget_timer *wt) @@ -1679,16 +1691,17 @@ wtimer_elapsed (struct wget_timer *wt) } /* Return the assessed granularity of the timer implementation, in - milliseconds. This is important for certain code that tries to - deal with "zero" time intervals. */ + milliseconds. This is used by code that tries to substitute a + better value for timers that have returned zero. */ double wtimer_granularity (void) { #ifdef TIMER_GETTIMEOFDAY - /* Granularity of gettimeofday is hugely architecture-dependent. - However, it appears that on modern machines it is better than - 1ms. Assume 100 usecs. */ + /* Granularity of gettimeofday varies wildly between architectures. + However, it appears that on modern machines it tends to be better + than 1ms. Assume 100 usecs. (Perhaps the configure process + could actually measure this?) */ return 0.1; #endif @@ -1698,7 +1711,8 @@ wtimer_granularity (void) #endif #ifdef TIMER_WINDOWS - /* #### Fill this in! */ + /* According to MSDN, GetSystemTime returns a broken-down time + structure the smallest member of which are milliseconds. */ return 1; #endif }