From 711bf72609e5d49ff7f5366a4664377cd3208d70 Mon Sep 17 00:00:00 2001 From: hniksic Date: Sat, 11 Oct 2003 06:57:11 -0700 Subject: [PATCH] [svn] Remove VERY_LONG_TYPE; use LARGE_INT instead. Remove special code for handling VERY_LONG_TYPE overflows. Make opt.quota a LARGE_INT. --- src/ChangeLog | 26 ++++++++++++++ src/ftp.c | 14 ++++---- src/http.c | 13 +++---- src/init.c | 99 ++++++++++++++++++++++++++++++++++----------------- src/main.c | 20 +++++------ src/options.h | 6 ++-- src/recur.c | 5 +-- src/retr.c | 39 +++----------------- src/retr.h | 3 -- src/sysdep.h | 48 ++++++++++++++----------- src/utils.c | 65 ++++++++++++--------------------- src/utils.h | 2 +- 12 files changed, 175 insertions(+), 165 deletions(-) diff --git a/src/ChangeLog b/src/ChangeLog index eaa9ec49..21a91547 100644 --- a/src/ChangeLog +++ b/src/ChangeLog @@ -1,3 +1,29 @@ +2003-10-11 Hrvoje Niksic + + * utils.c (large_int_to_string): Use snprintf() to print the + number. This will work even on systems where libc doesn't + understand %lld, but the compiler does, because it will use our + snprintf replacement. + + * init.c (parse_bytes_helper): New function. + (cmd_bytes): Use it to parse bytes, but cast the result to long. + (cmd_bytes_large): Ditto, but store the result to LARGE_INT. Used + for --quota so that --quota=10G works even on machines without + long long. + + * options.h (struct options): Declare quota as LARGE_INT. + + * retr.c (downloaded_exceeds_quota): Removed. + (downloaded_increase): Ditto. + (total_downloaded_bytes): New variable, replaces opt.downloaded, + which was the wrong place for it anyway. Updated callers of + downloaded_exceeds_quota and downloaded_increase to check this + variable directly. + + * sysdep.h: Get rid of VERY_LONG_TYPE. Use LARGE_INT for the same + purpose, defined as `long', `long long' or `double', depending on + size of long and whether long long is available. + 2003-10-11 Hrvoje Niksic * sysdep.h: Also check size of short for int32_t. diff --git a/src/ftp.c b/src/ftp.c index 5061d9cf..5a84a9d8 100644 --- a/src/ftp.c +++ b/src/ftp.c @@ -59,6 +59,8 @@ so, delete this exception statement from your version. */ extern int errno; #endif +extern LARGE_INT total_downloaded_bytes; + /* File where the "ls -al" listing will be saved. */ #define LIST_FILENAME ".listing" @@ -1195,7 +1197,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con) /* --dont-remove-listing was specified, so do count this towards the number of bytes and files downloaded. */ { - downloaded_increase (len); + total_downloaded_bytes += len; opt.numurls++; } @@ -1210,7 +1212,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con) downloaded if they're going to be deleted. People seeding proxies, for instance, may want to know how many bytes and files they've downloaded through it. */ - downloaded_increase (len); + total_downloaded_bytes += len; opt.numurls++; if (opt.delete_after) @@ -1336,7 +1338,7 @@ ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con) { char *old_target, *ofile; - if (downloaded_exceeds_quota ()) + if (opt.quota && total_downloaded_bytes > opt.quota) { --depth; return QUOTEXC; @@ -1540,7 +1542,7 @@ ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con) int size; char *odir, *newdir; - if (downloaded_exceeds_quota ()) + if (opt.quota && total_downloaded_bytes > opt.quota) break; if (f->type != FT_DIRECTORY) continue; @@ -1586,7 +1588,7 @@ Not descending to `%s' as it is excluded/not-included.\n"), newdir); /* Set the time-stamp? */ } - if (opt.quota && opt.downloaded > opt.quota) + if (opt.quota && total_downloaded_bytes > opt.quota) return QUOTEXC; else return RETROK; @@ -1704,7 +1706,7 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action) } } freefileinfo (start); - if (downloaded_exceeds_quota ()) + if (opt.quota && total_downloaded_bytes > opt.quota) return QUOTEXC; else /* #### Should we return `res' here? */ diff --git a/src/http.c b/src/http.c index 20010de0..4be6299d 100644 --- a/src/http.c +++ b/src/http.c @@ -53,6 +53,9 @@ so, delete this exception statement from your version. */ # include # endif #endif +#ifndef errno +extern int errno; +#endif #include "wget.h" #include "utils.h" @@ -73,10 +76,8 @@ so, delete this exception statement from your version. */ #include "convert.h" extern char *version_string; +extern LARGE_INT total_downloaded_bytes; -#ifndef errno -extern int errno; -#endif static int cookies_loaded_p; struct cookie_jar *wget_cookie_jar; @@ -1951,7 +1952,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); tms, u->url, hstat.len, hstat.contlen, locf, count); } ++opt.numurls; - downloaded_increase (hstat.len); + total_downloaded_bytes += hstat.len; /* Remember that we downloaded the file for later ".orig" code. */ if (*dt & ADDED_HTML_EXTENSION) @@ -1978,7 +1979,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); tms, u->url, hstat.len, locf, count); } ++opt.numurls; - downloaded_increase (hstat.len); + total_downloaded_bytes += hstat.len; /* Remember that we downloaded the file for later ".orig" code. */ if (*dt & ADDED_HTML_EXTENSION) @@ -2009,7 +2010,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size); "%s URL:%s [%ld/%ld] -> \"%s\" [%d]\n", tms, u->url, hstat.len, hstat.contlen, locf, count); ++opt.numurls; - downloaded_increase (hstat.len); + total_downloaded_bytes += hstat.len; /* Remember that we downloaded the file for later ".orig" code. */ if (*dt & ADDED_HTML_EXTENSION) diff --git a/src/init.c b/src/init.c index 0cd65ae5..50c66c44 100644 --- a/src/init.c +++ b/src/init.c @@ -84,6 +84,7 @@ static int enable_tilde_expansion; CMD_DECLARE (cmd_boolean); CMD_DECLARE (cmd_bytes); +CMD_DECLARE (cmd_bytes_large); CMD_DECLARE (cmd_directory_vector); CMD_DECLARE (cmd_lockable_boolean); CMD_DECLARE (cmd_number); @@ -184,7 +185,7 @@ static struct { { "proxypasswd", &opt.proxy_passwd, cmd_string }, { "proxyuser", &opt.proxy_user, cmd_string }, { "quiet", &opt.quiet, cmd_boolean }, - { "quota", &opt.quota, cmd_bytes }, + { "quota", &opt.quota, cmd_bytes_large }, { "randomwait", &opt.random_wait, cmd_boolean }, { "readtimeout", &opt.read_timeout, cmd_time }, { "reclevel", &opt.reclevel, cmd_number_inf }, @@ -849,60 +850,45 @@ cmd_directory_vector (const char *com, const char *val, void *closure) static int simple_atof PARAMS ((const char *, const char *, double *)); -/* Parse VAL as a number and set its value to CLOSURE (which should - point to a long int). - - By default, the value is assumed to be in bytes. If "K", "M", or - "G" are appended, the value is multiplied with 1<<10, 1<<20, or - 1<<30, respectively. Floating point values are allowed and are - cast to integer before use. The idea is to be able to use things - like 1.5k instead of "1536". - - The string "inf" is returned as 0. - - In case of error, 0 is returned and memory pointed to by CLOSURE - remains unmodified. */ +/* Enginge for cmd_bytes and cmd_bytes_large: converts a string such + as "100k" or "2.5G" to a floating point number. */ static int -cmd_bytes (const char *com, const char *val, void *closure) +parse_bytes_helper (const char *val, double *result) { - long mult; - double number; + double number, mult; const char *end = val + strlen (val); /* Check for "inf". */ if (0 == strcmp (val, "inf")) { - *(long *)closure = 0; + *result = 0; return 1; } /* Strip trailing whitespace. */ while (val < end && ISSPACE (end[-1])) --end; - if (val == end) - { - err: - fprintf (stderr, _("%s: %s: Invalid byte value `%s'\n"), - exec_name, com, val); - return 0; - } + return 0; switch (TOLOWER (end[-1])) { case 'k': - --end, mult = 1L<<10; + --end, mult = 1024.0; break; case 'm': - --end, mult = 1L<<20; + --end, mult = 1048576.0; break; case 'g': - --end, mult = 1L<<30; + --end, mult = 1073741824.0; + break; + case 't': + --end, mult = 1099511627776.0; break; default: - /* Not a recognized suffix: assume it belongs to the number. - (If not, atof simple_atof will raise an error.) */ + /* Not a recognized suffix: assume it's a digit. (If not, + simple_atof will raise an error.) */ mult = 1; } @@ -912,12 +898,59 @@ cmd_bytes (const char *com, const char *val, void *closure) while (val < end && ISSPACE (end[-1])) --end; if (val == end) - goto err; + return 0; if (!simple_atof (val, end, &number)) - goto err; + return 0; + + *result = number * mult; + return 1; +} - *(long *)closure = (long)(number * mult); +/* Parse VAL as a number and set its value to CLOSURE (which should + point to a long int). + + By default, the value is assumed to be in bytes. If "K", "M", or + "G" are appended, the value is multiplied with 1<<10, 1<<20, or + 1<<30, respectively. Floating point values are allowed and are + cast to integer before use. The idea is to be able to use things + like 1.5k instead of "1536". + + The string "inf" is returned as 0. + + In case of error, 0 is returned and memory pointed to by CLOSURE + remains unmodified. */ + +static int +cmd_bytes (const char *com, const char *val, void *closure) +{ + double byte_value; + if (!parse_bytes_helper (val, &byte_value)) + { + fprintf (stderr, _("%s: %s: Invalid byte value `%s'\n"), + exec_name, com, val); + return 0; + } + *(long *)closure = (long)byte_value; + return 1; +} + +/* Like cmd_bytes, but CLOSURE is interpreted as a pointer to + LARGE_INT. It works by converting the string to double, therefore + working with values up to 2^53-1 without loss of precision. This + value (8192 TB) is large enough to serve for a while. */ + +static int +cmd_bytes_large (const char *com, const char *val, void *closure) +{ + double byte_value; + if (!parse_bytes_helper (val, &byte_value)) + { + fprintf (stderr, _("%s: %s: Invalid byte value `%s'\n"), + exec_name, com, val); + return 0; + } + *(LARGE_INT *)closure = (LARGE_INT)byte_value; return 1; } diff --git a/src/main.c b/src/main.c index 59367ef7..bec8561d 100644 --- a/src/main.c +++ b/src/main.c @@ -50,6 +50,9 @@ so, delete this exception statement from your version. */ #endif /* HAVE_LOCALE_H */ #endif /* HAVE_NLS */ #include +#ifndef errno +extern int errno; +#endif #include "wget.h" #include "utils.h" @@ -73,14 +76,11 @@ so, delete this exception statement from your version. */ # define PATH_SEPARATOR '/' #endif -extern char *version_string; - -#ifndef errno -extern int errno; -#endif - struct options opt; +extern LARGE_INT total_downloaded_bytes; +extern char *version_string; + extern struct cookie_jar *wget_cookie_jar; /* From log.c. */ @@ -908,16 +908,14 @@ Can't timestamp and not clobber old files at the same time.\n")); /* Print the downloaded sum. */ if (opt.recursive || nurl > 1 - || (opt.input_filename && opt.downloaded != 0)) + || (opt.input_filename && total_downloaded_bytes != 0)) { logprintf (LOG_NOTQUIET, _("\nFINISHED --%s--\nDownloaded: %s bytes in %d files\n"), - time_str (NULL), - (opt.downloaded_overflow ? - "" : legible_very_long (opt.downloaded)), + time_str (NULL), legible_large_int (total_downloaded_bytes), opt.numurls); /* Print quota warning, if exceeded. */ - if (downloaded_exceeds_quota ()) + if (opt.quota && total_downloaded_bytes > opt.quota) logprintf (LOG_NOTQUIET, _("Download quota (%s bytes) EXCEEDED!\n"), legible (opt.quota)); diff --git a/src/options.h b/src/options.h index 9895e825..55ee968e 100644 --- a/src/options.h +++ b/src/options.h @@ -119,10 +119,8 @@ struct options long limit_rate; /* Limit the download rate to this many bps. */ - long quota; /* Maximum number of bytes to - retrieve. */ - VERY_LONG_TYPE downloaded; /* How much we downloaded already. */ - int downloaded_overflow; /* Whether the above overflowed. */ + LARGE_INT quota; /* Maximum file size to download and + store. */ int numurls; /* Number of successfully downloaded URLs */ diff --git a/src/recur.c b/src/recur.c index bf367074..d90aeadc 100644 --- a/src/recur.c +++ b/src/recur.c @@ -59,6 +59,7 @@ extern int errno; #endif extern char *version_string; +extern LARGE_INT total_downloaded_bytes; extern struct hash_table *dl_url_file_map; extern struct hash_table *downloaded_html_set; @@ -224,7 +225,7 @@ retrieve_tree (const char *start_url) int depth, html_allowed; boolean dash_p_leaf_HTML = FALSE; - if (downloaded_exceeds_quota ()) + if (opt.quota && total_downloaded_bytes > opt.quota) break; if (status == FWRITEERR) break; @@ -404,7 +405,7 @@ retrieve_tree (const char *start_url) url_free (start_url_parsed); string_set_free (blacklist); - if (downloaded_exceeds_quota ()) + if (opt.quota && total_downloaded_bytes > opt.quota) return QUOTEXC; else if (status == FWRITEERR) return FWRITEERR; diff --git a/src/retr.c b/src/retr.c index c0b439e8..35286111 100644 --- a/src/retr.c +++ b/src/retr.c @@ -66,6 +66,9 @@ extern int errno; /* See the comment in gethttp() why this is needed. */ int global_download_count; +/* Total size of downloaded files. Used to enforce quota. */ +LARGE_INT total_downloaded_bytes; + static struct { long chunk_bytes; @@ -576,7 +579,7 @@ retrieve_from_file (const char *file, int html, int *count) if (cur_url->ignore_when_downloading) continue; - if (downloaded_exceeds_quota ()) + if (opt.quota && total_downloaded_bytes > opt.quota) { status = QUOTEXC; break; @@ -614,40 +617,6 @@ printwhat (int n1, int n2) logputs (LOG_VERBOSE, (n1 == n2) ? _("Giving up.\n\n") : _("Retrying.\n\n")); } -/* Increment opt.downloaded by BY_HOW_MUCH. If an overflow occurs, - set opt.downloaded_overflow to 1. */ -void -downloaded_increase (unsigned long by_how_much) -{ - VERY_LONG_TYPE old; - if (opt.downloaded_overflow) - return; - old = opt.downloaded; - opt.downloaded += by_how_much; - if (opt.downloaded < old) /* carry flag, where are you when I - need you? */ - { - /* Overflow. */ - opt.downloaded_overflow = 1; - opt.downloaded = ~((VERY_LONG_TYPE)0); - } -} - -/* Return non-zero if the downloaded amount of bytes exceeds the - desired quota. If quota is not set or if the amount overflowed, 0 - is returned. */ -int -downloaded_exceeds_quota (void) -{ - if (!opt.quota) - return 0; - if (opt.downloaded_overflow) - /* We don't really know. (Wildly) assume not. */ - return 0; - - return opt.downloaded > opt.quota; -} - /* If opt.wait or opt.waitretry are specified, and if certain conditions are met, sleep the appropriate number of seconds. See the documentation of --wait and --waitretry for more information. diff --git a/src/retr.h b/src/retr.h index 2b804754..95090721 100644 --- a/src/retr.h +++ b/src/retr.h @@ -43,9 +43,6 @@ char *retr_rate PARAMS ((long, double, int)); double calc_rate PARAMS ((long, double, int *)); void printwhat PARAMS ((int, int)); -void downloaded_increase PARAMS ((unsigned long)); -int downloaded_exceeds_quota PARAMS ((void)); - void sleep_between_retrievals PARAMS ((int)); void rotate_backups PARAMS ((const char *)); diff --git a/src/sysdep.h b/src/sysdep.h index 05a0e773..5d74f48a 100644 --- a/src/sysdep.h +++ b/src/sysdep.h @@ -128,27 +128,30 @@ do { \ DEBUGP (("Closing fd %d\n", x)); \ } while (0) -/* Define a large ("very long") type useful for storing large - non-negative quantities that exceed sizes of normal download. Note - that this has nothing to do with large file support. For example, - one should be able to say `--quota=10G', large files - notwithstanding. - - On the machines where `long' is 64-bit, we use long. Otherwise, we - check whether `long long' is available and if yes, use that. If - long long is unavailable, we give up and just use `long'. - - Note: you cannot use VERY_LONG_TYPE along with printf(). When you - need to print it, use very_long_to_string(). */ - -#if SIZEOF_LONG >= 8 || SIZEOF_LONG_LONG == 0 -/* either long is "big enough", or long long is unavailable which - leaves long as the only choice. */ -# define VERY_LONG_TYPE unsigned long -#else /* use long long */ -/* long is smaller than 8 bytes, but long long is available. */ -# define VERY_LONG_TYPE unsigned long long -#endif /* use long long */ +/* Define a large integral type useful for storing large sizes that + exceed sizes of one download, such as when printing the sum of all + downloads. Note that this has nothing to do with large file + support, yet. + + We use a 64-bit integral type where available, `double' otherwise. + It's hard to print LARGE_INT's portably, but fortunately it's + rarely needed. */ + +#if SIZEOF_LONG >= 8 +/* Long is large enough: use it. */ +typedef long LARGE_INT; +# define LARGE_INT_FMT "%ld" +#else +# if SIZEOF_LONG_LONG == 8 +/* Long long is large enough: use it. */ +typedef long long LARGE_INT; +# define LARGE_INT_FMT "%lld" +# else +/* Use `double'. */ +typedef double LARGE_INT; +# define LARGE_INT_FMT "%.0f" +# endif +#endif /* These are defined in cmpt.c if missing, therefore it's generally safe to declare their parameters. */ @@ -167,6 +170,9 @@ char *strstr (); #ifndef HAVE_STRPTIME char *strptime (); #endif +#ifndef HAVE_SNPRINTF +int snprintf (); +#endif #ifndef HAVE_VSNPRINTF int vsnprintf (); #endif diff --git a/src/utils.c b/src/utils.c index c78b1708..85dd5fcd 100644 --- a/src/utils.c +++ b/src/utils.c @@ -1290,13 +1290,13 @@ free_keys_and_values (struct hash_table *ht) } -/* Engine for legible and legible_very_long; this function works on - strings. */ +/* Engine for legible and legible_large_int; add thousand separators + to numbers printed in strings. */ static char * legible_1 (const char *repr) { - static char outbuf[128]; + static char outbuf[48]; int i, i1, mod; char *outptr; const char *inptr; @@ -1304,7 +1304,9 @@ legible_1 (const char *repr) /* Reset the pointers. */ outptr = outbuf; inptr = repr; - /* If the number is negative, shift the pointers. */ + + /* Ignore the sign for the purpose of adding thousand + separators. */ if (*inptr == '-') { *outptr++ = '-'; @@ -1329,6 +1331,7 @@ legible_1 (const char *repr) } /* Legible -- return a static pointer to the legibly printed long. */ + char * legible (long l) { @@ -1338,53 +1341,29 @@ legible (long l) return legible_1 (inbuf); } -/* Write a string representation of NUMBER into the provided buffer. - We cannot use sprintf() because we cannot be sure whether the - platform supports printing of what we chose for VERY_LONG_TYPE. - - Example: Gcc supports `long long' under many platforms, but on many - of those the native libc knows nothing of it and therefore cannot - print it. - - How long BUFFER needs to be depends on the platform and the content - of NUMBER. For 64-bit VERY_LONG_TYPE (the most common case), 24 - bytes are sufficient. Using more might be a good idea. +/* Write a string representation of LARGE_INT NUMBER into the provided + buffer. The buffer should be able to accept 24 characters, + including the terminating zero. - This function does not go through the hoops that long_to_string - goes to because it doesn't aspire to be fast. (It's called perhaps - once in a Wget run.) */ + It would be dangerous to use sprintf, because the code wouldn't + work on a machine with gcc-provided long long support, but without + libc support for "%lld". However, such platforms will typically + not have snprintf and will use our version, which does support + "%lld" where long longs are available. */ static void -very_long_to_string (char *buffer, VERY_LONG_TYPE number) +large_int_to_string (char *buffer, LARGE_INT number) { - int i = 0; - int j; - - /* Print the number backwards... */ - do - { - buffer[i++] = '0' + number % 10; - number /= 10; - } - while (number); - - /* ...and reverse the order of the digits. */ - for (j = 0; j < i / 2; j++) - { - char c = buffer[j]; - buffer[j] = buffer[i - 1 - j]; - buffer[i - 1 - j] = c; - } - buffer[i] = '\0'; + snprintf (buffer, 24, LARGE_INT_FMT, number); } -/* The same as legible(), but works on VERY_LONG_TYPE. See sysdep.h. */ +/* The same as legible(), but works on LARGE_INT. */ + char * -legible_very_long (VERY_LONG_TYPE l) +legible_large_int (LARGE_INT l) { - char inbuf[128]; - /* Print the number into the buffer. */ - very_long_to_string (inbuf, l); + char inbuf[48]; + large_int_to_string (inbuf, l); return legible_1 (inbuf); } diff --git a/src/utils.h b/src/utils.h index 01a1d3de..d49847b7 100644 --- a/src/utils.h +++ b/src/utils.h @@ -102,7 +102,7 @@ void string_set_free PARAMS ((struct hash_table *)); void free_keys_and_values PARAMS ((struct hash_table *)); char *legible PARAMS ((long)); -char *legible_very_long PARAMS ((VERY_LONG_TYPE)); +char *legible_large_int PARAMS ((LARGE_INT)); int numdigit PARAMS ((long)); char *number_to_string PARAMS ((char *, long)); -- 2.39.2