for handling VERY_LONG_TYPE overflows.
Make opt.quota a LARGE_INT.
+2003-10-11 Hrvoje Niksic <hniksic@xemacs.org>
+
+ * utils.c (large_int_to_string): Use snprintf() to print the
+ number. This will work even on systems where libc doesn't
+ understand %lld, but the compiler does, because it will use our
+ snprintf replacement.
+
+ * init.c (parse_bytes_helper): New function.
+ (cmd_bytes): Use it to parse bytes, but cast the result to long.
+ (cmd_bytes_large): Ditto, but store the result to LARGE_INT. Used
+ for --quota so that --quota=10G works even on machines without
+ long long.
+
+ * options.h (struct options): Declare quota as LARGE_INT.
+
+ * retr.c (downloaded_exceeds_quota): Removed.
+ (downloaded_increase): Ditto.
+ (total_downloaded_bytes): New variable, replaces opt.downloaded,
+ which was the wrong place for it anyway. Updated callers of
+ downloaded_exceeds_quota and downloaded_increase to check this
+ variable directly.
+
+ * sysdep.h: Get rid of VERY_LONG_TYPE. Use LARGE_INT for the same
+ purpose, defined as `long', `long long' or `double', depending on
+ size of long and whether long long is available.
+
2003-10-11 Hrvoje Niksic <hniksic@xemacs.org>
* sysdep.h: Also check size of short for int32_t.
extern int errno;
#endif
+extern LARGE_INT total_downloaded_bytes;
+
/* File where the "ls -al" listing will be saved. */
#define LIST_FILENAME ".listing"
/* --dont-remove-listing was specified, so do count this towards the
number of bytes and files downloaded. */
{
- downloaded_increase (len);
+ total_downloaded_bytes += len;
opt.numurls++;
}
downloaded if they're going to be deleted. People seeding proxies,
for instance, may want to know how many bytes and files they've
downloaded through it. */
- downloaded_increase (len);
+ total_downloaded_bytes += len;
opt.numurls++;
if (opt.delete_after)
{
char *old_target, *ofile;
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
{
--depth;
return QUOTEXC;
int size;
char *odir, *newdir;
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
break;
if (f->type != FT_DIRECTORY)
continue;
/* Set the time-stamp? */
}
- if (opt.quota && opt.downloaded > opt.quota)
+ if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else
return RETROK;
}
}
freefileinfo (start);
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else
/* #### Should we return `res' here? */
# include <time.h>
# endif
#endif
+#ifndef errno
+extern int errno;
+#endif
#include "wget.h"
#include "utils.h"
#include "convert.h"
extern char *version_string;
+extern LARGE_INT total_downloaded_bytes;
-#ifndef errno
-extern int errno;
-#endif
\f
static int cookies_loaded_p;
struct cookie_jar *wget_cookie_jar;
tms, u->url, hstat.len, hstat.contlen, locf, count);
}
++opt.numurls;
- downloaded_increase (hstat.len);
+ total_downloaded_bytes += hstat.len;
/* Remember that we downloaded the file for later ".orig" code. */
if (*dt & ADDED_HTML_EXTENSION)
tms, u->url, hstat.len, locf, count);
}
++opt.numurls;
- downloaded_increase (hstat.len);
+ total_downloaded_bytes += hstat.len;
/* Remember that we downloaded the file for later ".orig" code. */
if (*dt & ADDED_HTML_EXTENSION)
"%s URL:%s [%ld/%ld] -> \"%s\" [%d]\n",
tms, u->url, hstat.len, hstat.contlen, locf, count);
++opt.numurls;
- downloaded_increase (hstat.len);
+ total_downloaded_bytes += hstat.len;
/* Remember that we downloaded the file for later ".orig" code. */
if (*dt & ADDED_HTML_EXTENSION)
CMD_DECLARE (cmd_boolean);
CMD_DECLARE (cmd_bytes);
+CMD_DECLARE (cmd_bytes_large);
CMD_DECLARE (cmd_directory_vector);
CMD_DECLARE (cmd_lockable_boolean);
CMD_DECLARE (cmd_number);
{ "proxypasswd", &opt.proxy_passwd, cmd_string },
{ "proxyuser", &opt.proxy_user, cmd_string },
{ "quiet", &opt.quiet, cmd_boolean },
- { "quota", &opt.quota, cmd_bytes },
+ { "quota", &opt.quota, cmd_bytes_large },
{ "randomwait", &opt.random_wait, cmd_boolean },
{ "readtimeout", &opt.read_timeout, cmd_time },
{ "reclevel", &opt.reclevel, cmd_number_inf },
static int simple_atof PARAMS ((const char *, const char *, double *));
-/* Parse VAL as a number and set its value to CLOSURE (which should
- point to a long int).
-
- By default, the value is assumed to be in bytes. If "K", "M", or
- "G" are appended, the value is multiplied with 1<<10, 1<<20, or
- 1<<30, respectively. Floating point values are allowed and are
- cast to integer before use. The idea is to be able to use things
- like 1.5k instead of "1536".
-
- The string "inf" is returned as 0.
-
- In case of error, 0 is returned and memory pointed to by CLOSURE
- remains unmodified. */
+/* Enginge for cmd_bytes and cmd_bytes_large: converts a string such
+ as "100k" or "2.5G" to a floating point number. */
static int
-cmd_bytes (const char *com, const char *val, void *closure)
+parse_bytes_helper (const char *val, double *result)
{
- long mult;
- double number;
+ double number, mult;
const char *end = val + strlen (val);
/* Check for "inf". */
if (0 == strcmp (val, "inf"))
{
- *(long *)closure = 0;
+ *result = 0;
return 1;
}
/* Strip trailing whitespace. */
while (val < end && ISSPACE (end[-1]))
--end;
-
if (val == end)
- {
- err:
- fprintf (stderr, _("%s: %s: Invalid byte value `%s'\n"),
- exec_name, com, val);
- return 0;
- }
+ return 0;
switch (TOLOWER (end[-1]))
{
case 'k':
- --end, mult = 1L<<10;
+ --end, mult = 1024.0;
break;
case 'm':
- --end, mult = 1L<<20;
+ --end, mult = 1048576.0;
break;
case 'g':
- --end, mult = 1L<<30;
+ --end, mult = 1073741824.0;
+ break;
+ case 't':
+ --end, mult = 1099511627776.0;
break;
default:
- /* Not a recognized suffix: assume it belongs to the number.
- (If not, atof simple_atof will raise an error.) */
+ /* Not a recognized suffix: assume it's a digit. (If not,
+ simple_atof will raise an error.) */
mult = 1;
}
while (val < end && ISSPACE (end[-1]))
--end;
if (val == end)
- goto err;
+ return 0;
if (!simple_atof (val, end, &number))
- goto err;
+ return 0;
+
+ *result = number * mult;
+ return 1;
+}
- *(long *)closure = (long)(number * mult);
+/* Parse VAL as a number and set its value to CLOSURE (which should
+ point to a long int).
+
+ By default, the value is assumed to be in bytes. If "K", "M", or
+ "G" are appended, the value is multiplied with 1<<10, 1<<20, or
+ 1<<30, respectively. Floating point values are allowed and are
+ cast to integer before use. The idea is to be able to use things
+ like 1.5k instead of "1536".
+
+ The string "inf" is returned as 0.
+
+ In case of error, 0 is returned and memory pointed to by CLOSURE
+ remains unmodified. */
+
+static int
+cmd_bytes (const char *com, const char *val, void *closure)
+{
+ double byte_value;
+ if (!parse_bytes_helper (val, &byte_value))
+ {
+ fprintf (stderr, _("%s: %s: Invalid byte value `%s'\n"),
+ exec_name, com, val);
+ return 0;
+ }
+ *(long *)closure = (long)byte_value;
+ return 1;
+}
+
+/* Like cmd_bytes, but CLOSURE is interpreted as a pointer to
+ LARGE_INT. It works by converting the string to double, therefore
+ working with values up to 2^53-1 without loss of precision. This
+ value (8192 TB) is large enough to serve for a while. */
+
+static int
+cmd_bytes_large (const char *com, const char *val, void *closure)
+{
+ double byte_value;
+ if (!parse_bytes_helper (val, &byte_value))
+ {
+ fprintf (stderr, _("%s: %s: Invalid byte value `%s'\n"),
+ exec_name, com, val);
+ return 0;
+ }
+ *(LARGE_INT *)closure = (LARGE_INT)byte_value;
return 1;
}
#endif /* HAVE_LOCALE_H */
#endif /* HAVE_NLS */
#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
#include "wget.h"
#include "utils.h"
# define PATH_SEPARATOR '/'
#endif
-extern char *version_string;
-
-#ifndef errno
-extern int errno;
-#endif
-
struct options opt;
+extern LARGE_INT total_downloaded_bytes;
+extern char *version_string;
+
extern struct cookie_jar *wget_cookie_jar;
/* From log.c. */
/* Print the downloaded sum. */
if (opt.recursive
|| nurl > 1
- || (opt.input_filename && opt.downloaded != 0))
+ || (opt.input_filename && total_downloaded_bytes != 0))
{
logprintf (LOG_NOTQUIET,
_("\nFINISHED --%s--\nDownloaded: %s bytes in %d files\n"),
- time_str (NULL),
- (opt.downloaded_overflow ?
- "<overflow>" : legible_very_long (opt.downloaded)),
+ time_str (NULL), legible_large_int (total_downloaded_bytes),
opt.numurls);
/* Print quota warning, if exceeded. */
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
logprintf (LOG_NOTQUIET,
_("Download quota (%s bytes) EXCEEDED!\n"),
legible (opt.quota));
long limit_rate; /* Limit the download rate to this
many bps. */
- long quota; /* Maximum number of bytes to
- retrieve. */
- VERY_LONG_TYPE downloaded; /* How much we downloaded already. */
- int downloaded_overflow; /* Whether the above overflowed. */
+ LARGE_INT quota; /* Maximum file size to download and
+ store. */
int numurls; /* Number of successfully downloaded
URLs */
#endif
extern char *version_string;
+extern LARGE_INT total_downloaded_bytes;
extern struct hash_table *dl_url_file_map;
extern struct hash_table *downloaded_html_set;
int depth, html_allowed;
boolean dash_p_leaf_HTML = FALSE;
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
break;
if (status == FWRITEERR)
break;
url_free (start_url_parsed);
string_set_free (blacklist);
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else if (status == FWRITEERR)
return FWRITEERR;
/* See the comment in gethttp() why this is needed. */
int global_download_count;
+/* Total size of downloaded files. Used to enforce quota. */
+LARGE_INT total_downloaded_bytes;
+
\f
static struct {
long chunk_bytes;
if (cur_url->ignore_when_downloading)
continue;
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
{
status = QUOTEXC;
break;
logputs (LOG_VERBOSE, (n1 == n2) ? _("Giving up.\n\n") : _("Retrying.\n\n"));
}
-/* Increment opt.downloaded by BY_HOW_MUCH. If an overflow occurs,
- set opt.downloaded_overflow to 1. */
-void
-downloaded_increase (unsigned long by_how_much)
-{
- VERY_LONG_TYPE old;
- if (opt.downloaded_overflow)
- return;
- old = opt.downloaded;
- opt.downloaded += by_how_much;
- if (opt.downloaded < old) /* carry flag, where are you when I
- need you? */
- {
- /* Overflow. */
- opt.downloaded_overflow = 1;
- opt.downloaded = ~((VERY_LONG_TYPE)0);
- }
-}
-
-/* Return non-zero if the downloaded amount of bytes exceeds the
- desired quota. If quota is not set or if the amount overflowed, 0
- is returned. */
-int
-downloaded_exceeds_quota (void)
-{
- if (!opt.quota)
- return 0;
- if (opt.downloaded_overflow)
- /* We don't really know. (Wildly) assume not. */
- return 0;
-
- return opt.downloaded > opt.quota;
-}
-
/* If opt.wait or opt.waitretry are specified, and if certain
conditions are met, sleep the appropriate number of seconds. See
the documentation of --wait and --waitretry for more information.
double calc_rate PARAMS ((long, double, int *));
void printwhat PARAMS ((int, int));
-void downloaded_increase PARAMS ((unsigned long));
-int downloaded_exceeds_quota PARAMS ((void));
-
void sleep_between_retrievals PARAMS ((int));
void rotate_backups PARAMS ((const char *));
DEBUGP (("Closing fd %d\n", x)); \
} while (0)
-/* Define a large ("very long") type useful for storing large
- non-negative quantities that exceed sizes of normal download. Note
- that this has nothing to do with large file support. For example,
- one should be able to say `--quota=10G', large files
- notwithstanding.
-
- On the machines where `long' is 64-bit, we use long. Otherwise, we
- check whether `long long' is available and if yes, use that. If
- long long is unavailable, we give up and just use `long'.
-
- Note: you cannot use VERY_LONG_TYPE along with printf(). When you
- need to print it, use very_long_to_string(). */
-
-#if SIZEOF_LONG >= 8 || SIZEOF_LONG_LONG == 0
-/* either long is "big enough", or long long is unavailable which
- leaves long as the only choice. */
-# define VERY_LONG_TYPE unsigned long
-#else /* use long long */
-/* long is smaller than 8 bytes, but long long is available. */
-# define VERY_LONG_TYPE unsigned long long
-#endif /* use long long */
+/* Define a large integral type useful for storing large sizes that
+ exceed sizes of one download, such as when printing the sum of all
+ downloads. Note that this has nothing to do with large file
+ support, yet.
+
+ We use a 64-bit integral type where available, `double' otherwise.
+ It's hard to print LARGE_INT's portably, but fortunately it's
+ rarely needed. */
+
+#if SIZEOF_LONG >= 8
+/* Long is large enough: use it. */
+typedef long LARGE_INT;
+# define LARGE_INT_FMT "%ld"
+#else
+# if SIZEOF_LONG_LONG == 8
+/* Long long is large enough: use it. */
+typedef long long LARGE_INT;
+# define LARGE_INT_FMT "%lld"
+# else
+/* Use `double'. */
+typedef double LARGE_INT;
+# define LARGE_INT_FMT "%.0f"
+# endif
+#endif
/* These are defined in cmpt.c if missing, therefore it's generally
safe to declare their parameters. */
#ifndef HAVE_STRPTIME
char *strptime ();
#endif
+#ifndef HAVE_SNPRINTF
+int snprintf ();
+#endif
#ifndef HAVE_VSNPRINTF
int vsnprintf ();
#endif
}
\f
-/* Engine for legible and legible_very_long; this function works on
- strings. */
+/* Engine for legible and legible_large_int; add thousand separators
+ to numbers printed in strings. */
static char *
legible_1 (const char *repr)
{
- static char outbuf[128];
+ static char outbuf[48];
int i, i1, mod;
char *outptr;
const char *inptr;
/* Reset the pointers. */
outptr = outbuf;
inptr = repr;
- /* If the number is negative, shift the pointers. */
+
+ /* Ignore the sign for the purpose of adding thousand
+ separators. */
if (*inptr == '-')
{
*outptr++ = '-';
}
/* Legible -- return a static pointer to the legibly printed long. */
+
char *
legible (long l)
{
return legible_1 (inbuf);
}
-/* Write a string representation of NUMBER into the provided buffer.
- We cannot use sprintf() because we cannot be sure whether the
- platform supports printing of what we chose for VERY_LONG_TYPE.
-
- Example: Gcc supports `long long' under many platforms, but on many
- of those the native libc knows nothing of it and therefore cannot
- print it.
-
- How long BUFFER needs to be depends on the platform and the content
- of NUMBER. For 64-bit VERY_LONG_TYPE (the most common case), 24
- bytes are sufficient. Using more might be a good idea.
+/* Write a string representation of LARGE_INT NUMBER into the provided
+ buffer. The buffer should be able to accept 24 characters,
+ including the terminating zero.
- This function does not go through the hoops that long_to_string
- goes to because it doesn't aspire to be fast. (It's called perhaps
- once in a Wget run.) */
+ It would be dangerous to use sprintf, because the code wouldn't
+ work on a machine with gcc-provided long long support, but without
+ libc support for "%lld". However, such platforms will typically
+ not have snprintf and will use our version, which does support
+ "%lld" where long longs are available. */
static void
-very_long_to_string (char *buffer, VERY_LONG_TYPE number)
+large_int_to_string (char *buffer, LARGE_INT number)
{
- int i = 0;
- int j;
-
- /* Print the number backwards... */
- do
- {
- buffer[i++] = '0' + number % 10;
- number /= 10;
- }
- while (number);
-
- /* ...and reverse the order of the digits. */
- for (j = 0; j < i / 2; j++)
- {
- char c = buffer[j];
- buffer[j] = buffer[i - 1 - j];
- buffer[i - 1 - j] = c;
- }
- buffer[i] = '\0';
+ snprintf (buffer, 24, LARGE_INT_FMT, number);
}
-/* The same as legible(), but works on VERY_LONG_TYPE. See sysdep.h. */
+/* The same as legible(), but works on LARGE_INT. */
+
char *
-legible_very_long (VERY_LONG_TYPE l)
+legible_large_int (LARGE_INT l)
{
- char inbuf[128];
- /* Print the number into the buffer. */
- very_long_to_string (inbuf, l);
+ char inbuf[48];
+ large_int_to_string (inbuf, l);
return legible_1 (inbuf);
}
void free_keys_and_values PARAMS ((struct hash_table *));
char *legible PARAMS ((long));
-char *legible_very_long PARAMS ((VERY_LONG_TYPE));
+char *legible_large_int PARAMS ((LARGE_INT));
int numdigit PARAMS ((long));
char *number_to_string PARAMS ((char *, long));
projects / wget / commitdiff