X-Git-Url: http://sjero.net/git/?p=wget;a=blobdiff_plain;f=src%2Futils.c;h=dc16d55afc5940d095b9ada5d33f4d6d77deb809;hp=fca15800248351052a573cb2582eb8eb466a91a1;hb=2219d47ba301c3ea47b36291dda8eabead0fc75d;hpb=cdcf67a5bdae9c56d263ebf7608b52701851cf22 diff --git a/src/utils.c b/src/utils.c index fca15800..dc16d55a 100644 --- a/src/utils.c +++ b/src/utils.c @@ -1,6 +1,5 @@ -/* Various functions of utilitarian nature. - Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001 - Free Software Foundation, Inc. +/* Various utility functions. + Copyright (C) 2005 Free Software Foundation, Inc. This file is part of GNU Wget. @@ -16,7 +15,17 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with Wget; if not, write to the Free Software -Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ +Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +In addition, as a special exception, the Free Software Foundation +gives permission to link the code of its release of Wget with the +OpenSSL project's "OpenSSL" library (or with modified versions of it +that use the same license as the "OpenSSL" library), and distribute +the linked executables. You must obey the GNU General Public License +in all respects for all of the code used other than "OpenSSL". If you +modify this file, you may extend this exception to your version of the +file, but you are not obligated to do so. If you do not wish to do +so, delete this exception statement from your version. */ #include @@ -37,7 +46,9 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifdef HAVE_PWD_H # include #endif -#include +#ifdef HAVE_LIMITS_H +# include +#endif #ifdef HAVE_UTIME_H # include #endif @@ -50,263 +61,65 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #endif #include #include +#ifdef WGET_USE_STDARG +# include +#else +# include +#endif + +/* For TIOCGWINSZ and friends: */ #ifdef HAVE_SYS_IOCTL_H # include #endif +#ifdef HAVE_TERMIOS_H +# include +#endif + +/* Needed for run_with_timeout. */ +#undef USE_SIGNAL_TIMEOUT +#ifdef HAVE_SIGNAL_H +# include +#endif +#ifdef HAVE_SETJMP_H +# include +#endif + +#ifndef HAVE_SIGSETJMP +/* If sigsetjmp is a macro, configure won't pick it up. */ +# ifdef sigsetjmp +# define HAVE_SIGSETJMP +# endif +#endif + +#ifdef HAVE_SIGNAL +# ifdef HAVE_SIGSETJMP +# define USE_SIGNAL_TIMEOUT +# endif +# ifdef HAVE_SIGBLOCK +# define USE_SIGNAL_TIMEOUT +# endif +#endif #include "wget.h" #include "utils.h" -#include "fnmatch.h" #include "hash.h" #ifndef errno extern int errno; #endif -/* This section implements several wrappers around the basic - allocation routines. This is done for two reasons: first, so that - the callers of these functions need not consistently check for - errors. If there is not enough virtual memory for running Wget, - something is seriously wrong, and Wget exits with an appropriate - error message. - - The second reason why these are useful is that, if DEBUG_MALLOC is - defined, they also provide a handy (if crude) malloc debugging - interface that checks memory leaks. */ - -/* Croak the fatal memory error and bail out with non-zero exit - status. */ -static void -memfatal (const char *what) -{ - /* HACK: expose save_log_p from log.c, so we can turn it off in - order to prevent saving the log. Saving the log is dangerous - because logprintf() and logputs() can call malloc(), so this - could infloop. When logging is turned off, infloop can no longer - happen. - - #### This is no longer really necessary because the new routines - in log.c cons only if the line exceeds eighty characters. But - this can come at the end of a line, so it's OK to be careful. - - On a more serious note, it would be good to have a - log_forced_shutdown() routine that exposes this cleanly. */ - extern int save_log_p; - - save_log_p = 0; - logprintf (LOG_ALWAYS, _("%s: %s: Not enough memory.\n"), exec_name, what); - exit (1); -} - -/* These functions end with _real because they need to be - distinguished from the debugging functions, and from the macros. - Explanation follows: - - If memory debugging is not turned on, wget.h defines these: - - #define xmalloc xmalloc_real - #define xrealloc xrealloc_real - #define xstrdup xstrdup_real - #define xfree free - - In case of memory debugging, the definitions are a bit more - complex, because we want to provide more information, *and* we want - to call the debugging code. (The former is the reason why xmalloc - and friends need to be macros in the first place.) Then it looks - like this: - - #define xmalloc(a) xmalloc_debug (a, __FILE__, __LINE__) - #define xfree(a) xfree_debug (a, __FILE__, __LINE__) - #define xrealloc(a, b) xrealloc_debug (a, b, __FILE__, __LINE__) - #define xstrdup(a) xstrdup_debug (a, __FILE__, __LINE__) - - Each of the *_debug function does its magic and calls the real one. */ - -#ifdef DEBUG_MALLOC -# define STATIC_IF_DEBUG static -#else -# define STATIC_IF_DEBUG -#endif - -STATIC_IF_DEBUG void * -xmalloc_real (size_t size) -{ - void *ptr = malloc (size); - if (!ptr) - memfatal ("malloc"); - return ptr; -} - -STATIC_IF_DEBUG void * -xrealloc_real (void *ptr, size_t newsize) -{ - void *newptr; - - /* Not all Un*xes have the feature of realloc() that calling it with - a NULL-pointer is the same as malloc(), but it is easy to - simulate. */ - if (ptr) - newptr = realloc (ptr, newsize); - else - newptr = malloc (newsize); - if (!newptr) - memfatal ("realloc"); - return newptr; -} - -STATIC_IF_DEBUG char * -xstrdup_real (const char *s) -{ - char *copy; - -#ifndef HAVE_STRDUP - int l = strlen (s); - copy = malloc (l + 1); - if (!copy) - memfatal ("strdup"); - memcpy (copy, s, l + 1); -#else /* HAVE_STRDUP */ - copy = strdup (s); - if (!copy) - memfatal ("strdup"); -#endif /* HAVE_STRDUP */ - - return copy; -} - -#ifdef DEBUG_MALLOC - -/* Crude home-grown routines for debugging some malloc-related - problems. Featured: - - * Counting the number of malloc and free invocations, and reporting - the "balance", i.e. how many times more malloc was called than it - was the case with free. - - * Making malloc store its entry into a simple array and free remove - stuff from that array. At the end, print the pointers which have - not been freed, along with the source file and the line number. - This also has the side-effect of detecting freeing memory that - was never allocated. - - Note that this kind of memory leak checking strongly depends on - every malloc() being followed by a free(), even if the program is - about to finish. Wget is careful to free the data structure it - allocated in init.c. */ - -static int malloc_count, free_count; - -static struct { - char *ptr; - const char *file; - int line; -} malloc_debug[100000]; - -/* Both register_ptr and unregister_ptr take O(n) operations to run, - which can be a real problem. It would be nice to use a hash table - for malloc_debug, but the functions in hash.c are not suitable - because they can call malloc() themselves. Maybe it would work if - the hash table were preallocated to a huge size, and if we set the - rehash threshold to 1.0. */ - -/* Register PTR in malloc_debug. Abort if this is not possible - (presumably due to the number of current allocations exceeding the - size of malloc_debug.) */ - -static void -register_ptr (void *ptr, const char *file, int line) -{ - int i; - for (i = 0; i < ARRAY_SIZE (malloc_debug); i++) - if (malloc_debug[i].ptr == NULL) - { - malloc_debug[i].ptr = ptr; - malloc_debug[i].file = file; - malloc_debug[i].line = line; - return; - } - abort (); -} - -/* Unregister PTR from malloc_debug. Abort if PTR is not present in - malloc_debug. (This catches calling free() with a bogus pointer.) */ - -static void -unregister_ptr (void *ptr) -{ - int i; - for (i = 0; i < ARRAY_SIZE (malloc_debug); i++) - if (malloc_debug[i].ptr == ptr) - { - malloc_debug[i].ptr = NULL; - return; - } - abort (); -} - -/* Print the malloc debug stats that can be gathered from the above - information. Currently this is the count of mallocs, frees, the - difference between the two, and the dump of the contents of - malloc_debug. The last part are the memory leaks. */ - -void -print_malloc_debug_stats (void) -{ - int i; - printf ("\nMalloc: %d\nFree: %d\nBalance: %d\n\n", - malloc_count, free_count, malloc_count - free_count); - for (i = 0; i < ARRAY_SIZE (malloc_debug); i++) - if (malloc_debug[i].ptr != NULL) - printf ("0x%08ld: %s:%d\n", (long)malloc_debug[i].ptr, - malloc_debug[i].file, malloc_debug[i].line); -} - -void * -xmalloc_debug (size_t size, const char *source_file, int source_line) -{ - void *ptr = xmalloc_real (size); - ++malloc_count; - register_ptr (ptr, source_file, source_line); - return ptr; -} - -void -xfree_debug (void *ptr, const char *source_file, int source_line) -{ - assert (ptr != NULL); - ++free_count; - unregister_ptr (ptr); - free (ptr); -} - -void * -xrealloc_debug (void *ptr, size_t newsize, const char *source_file, int source_line) -{ - void *newptr = xrealloc_real (ptr, newsize); - if (!ptr) - { - ++malloc_count; - register_ptr (newptr, source_file, source_line); - } - else if (newptr != ptr) - { - unregister_ptr (ptr); - register_ptr (newptr, source_file, source_line); - } - return newptr; -} +/* Utility function: like xstrdup(), but also lowercases S. */ char * -xstrdup_debug (const char *s, const char *source_file, int source_line) +xstrdup_lower (const char *s) { - char *copy = xstrdup_real (s); - ++malloc_count; - register_ptr (copy, source_file, source_line); + char *copy = xstrdup (s); + char *p = copy; + for (; *p; p++) + *p = TOLOWER (*p); return copy; } -#endif /* DEBUG_MALLOC */ - /* Copy the string formed by two pointers (one on the beginning, other on the char after the last char) to a new, malloc-ed location. 0-terminate it. */ @@ -355,23 +168,111 @@ sepstring (const char *s) return res; } +#ifdef WGET_USE_STDARG +# define VA_START(args, arg1) va_start (args, arg1) +#else +# define VA_START(args, ignored) va_start (args) +#endif + +/* Like sprintf, but allocates a string of sufficient size with malloc + and returns it. GNU libc has a similar function named asprintf, + which requires the pointer to the string to be passed. */ + +char * +aprintf (const char *fmt, ...) +{ + /* This function is implemented using vsnprintf, which we provide + for the systems that don't have it. Therefore, it should be 100% + portable. */ + + int size = 32; + char *str = xmalloc (size); + + while (1) + { + int n; + va_list args; + + /* See log_vprintf_internal for explanation why it's OK to rely + on the return value of vsnprintf. */ + + VA_START (args, fmt); + n = vsnprintf (str, size, fmt, args); + va_end (args); + + /* If the printing worked, return the string. */ + if (n > -1 && n < size) + return str; + + /* Else try again with a larger buffer. */ + if (n > -1) /* C99 */ + size = n + 1; /* precisely what is needed */ + else + size <<= 1; /* twice the old size */ + str = xrealloc (str, size); + } + return NULL; /* unreached */ +} + +/* Concatenate the NULL-terminated list of string arguments into + freshly allocated space. */ + +char * +concat_strings (const char *str0, ...) +{ + va_list args; + int saved_lengths[5]; /* inspired by Apache's apr_pstrcat */ + char *ret, *p; + + const char *next_str; + int total_length = 0; + int argcount; + + /* Calculate the length of and allocate the resulting string. */ + + argcount = 0; + VA_START (args, str0); + for (next_str = str0; next_str != NULL; next_str = va_arg (args, char *)) + { + int len = strlen (next_str); + if (argcount < countof (saved_lengths)) + saved_lengths[argcount++] = len; + total_length += len; + } + va_end (args); + p = ret = xmalloc (total_length + 1); + + /* Copy the strings into the allocated space. */ + + argcount = 0; + VA_START (args, str0); + for (next_str = str0; next_str != NULL; next_str = va_arg (args, char *)) + { + int len; + if (argcount < countof (saved_lengths)) + len = saved_lengths[argcount++]; + else + len = strlen (next_str); + memcpy (p, next_str, len); + p += len; + } + va_end (args); + *p = '\0'; + + return ret; +} + /* Return pointer to a static char[] buffer in which zero-terminated string-representation of TM (in form hh:mm:ss) is printed. - If TM is non-NULL, the current time-in-seconds will be stored - there. - - (#### This is misleading: one would expect TM would be used instead - of the current time in that case. This design was probably - influenced by the design time(2), and should be changed at some - points. No callers use non-NULL TM anyway.) */ + If TM is NULL, the current time will be used. */ char * time_str (time_t *tm) { static char output[15]; struct tm *ptm; - time_t secs = time (tm); + time_t secs = tm ? *tm : time (NULL); if (secs == -1) { @@ -392,7 +293,7 @@ datetime_str (time_t *tm) { static char output[20]; /* "YYYY-MM-DD hh:mm:ss" + \0 */ struct tm *ptm; - time_t secs = time (tm); + time_t secs = tm ? *tm : time (NULL); if (secs == -1) { @@ -417,12 +318,21 @@ fork_to_background (void) { pid_t pid; /* Whether we arrange our own version of opt.lfilename here. */ - int changedp = 0; + int logfile_changed = 0; if (!opt.lfilename) { - opt.lfilename = unique_name (DEFAULT_LOGFILE); - changedp = 1; + /* We must create the file immediately to avoid either a race + condition (which arises from using unique_name and failing to + use fopen_excl) or lying to the user about the log file name + (which arises from using unique_name, printing the name, and + using fopen_excl later on.) */ + FILE *new_log_fp = unique_create (DEFAULT_LOGFILE, 0, &opt.lfilename); + if (new_log_fp) + { + logfile_changed = 1; + fclose (new_log_fp); + } } pid = fork (); if (pid < 0) @@ -434,135 +344,19 @@ fork_to_background (void) else if (pid != 0) { /* parent, no error */ - printf (_("Continuing in background.\n")); - if (changedp) + printf (_("Continuing in background, pid %d.\n"), (int)pid); + if (logfile_changed) printf (_("Output will be written to `%s'.\n"), opt.lfilename); - exit (0); - } - /* child: keep running */ -} -#endif /* not WINDOWS */ - -char * -ps (char *orig) -{ - char *r = xstrdup (orig); - path_simplify (r); - return r; -} - -/* Canonicalize PATH, and return a new path. The new path differs from PATH - in that: - Multple `/'s are collapsed to a single `/'. - Leading `./'s and trailing `/.'s are removed. - Trailing `/'s are removed. - Non-leading `../'s and trailing `..'s are handled by removing - portions of the path. - - E.g. "a/b/c/./../d/.." will yield "a/b". This function originates - from GNU Bash. - - Changes for Wget: - Always use '/' as stub_char. - Don't check for local things using canon_stat. - Change the original string instead of strdup-ing. - React correctly when beginning with `./' and `../'. - Don't zip out trailing slashes. */ -void -path_simplify (char *path) -{ - register int i, start, ddot; - char stub_char; - - if (!*path) - return; - - /*stub_char = (*path == '/') ? '/' : '.';*/ - stub_char = '/'; - - /* Addition: Remove all `./'-s preceding the string. If `../'-s - precede, put `/' in front and remove them too. */ - i = 0; - ddot = 0; - while (1) - { - if (path[i] == '.' && path[i + 1] == '/') - i += 2; - else if (path[i] == '.' && path[i + 1] == '.' && path[i + 2] == '/') - { - i += 3; - ddot = 1; - } - else - break; + exit (0); /* #### should we use _exit()? */ } - if (i) - strcpy (path, path + i - ddot); - /* Replace single `.' or `..' with `/'. */ - if ((path[0] == '.' && path[1] == '\0') - || (path[0] == '.' && path[1] == '.' && path[2] == '\0')) - { - path[0] = stub_char; - path[1] = '\0'; - return; - } - /* Walk along PATH looking for things to compact. */ - i = 0; - while (1) - { - if (!path[i]) - break; - - while (path[i] && path[i] != '/') - i++; - - start = i++; - - /* If we didn't find any slashes, then there is nothing left to do. */ - if (!path[start]) - break; - - /* Handle multiple `/'s in a row. */ - while (path[i] == '/') - i++; - - if ((start + 1) != i) - { - strcpy (path + start + 1, path + i); - i = start + 1; - } - - /* Check for `../', `./' or trailing `.' by itself. */ - if (path[i] == '.') - { - /* Handle trailing `.' by itself. */ - if (!path[i + 1]) - { - path[--i] = '\0'; - break; - } - - /* Handle `./'. */ - if (path[i + 1] == '/') - { - strcpy (path + i, path + i + 1); - i = (start < 0) ? 0 : start; - continue; - } - - /* Handle `../' or trailing `..' by itself. */ - if (path[i + 1] == '.' && - (path[i + 2] == '/' || !path[i + 2])) - { - while (--start > -1 && path[start] != '/'); - strcpy (path + start + 1, path + i + 2); - i = (start < 0) ? 0 : start; - continue; - } - } /* path == '.' */ - } /* while */ + /* child: give up the privileges and keep running. */ + setsid (); + freopen ("/dev/null", "r", stdin); + freopen ("/dev/null", "w", stdout); + freopen ("/dev/null", "w", stderr); } +#endif /* not WINDOWS */ /* "Touch" FILE, i.e. make its atime and mtime equal to the time specified with TM. */ @@ -587,7 +381,7 @@ int remove_link (const char *file) { int err = 0; - struct stat st; + struct_stat st; if (lstat (file, &st) == 0 && S_ISLNK (st.st_mode)) { @@ -613,7 +407,7 @@ file_exists_p (const char *filename) #ifdef HAVE_ACCESS return access (filename, F_OK) >= 0; #else - struct stat buf; + struct_stat buf; return stat (filename, &buf) >= 0; #endif } @@ -623,7 +417,7 @@ file_exists_p (const char *filename) int file_non_directory_p (const char *path) { - struct stat buf; + struct_stat buf; /* Use lstat() rather than stat() so that symbolic links pointing to directories can be identified correctly. */ if (lstat (path, &buf) != 0) @@ -631,39 +425,149 @@ file_non_directory_p (const char *path) return S_ISDIR (buf.st_mode) ? 0 : 1; } -/* Return a unique filename, given a prefix and count */ +/* Return the size of file named by FILENAME, or -1 if it cannot be + opened or seeked into. */ +wgint +file_size (const char *filename) +{ +#if defined(HAVE_FSEEKO) && defined(HAVE_FTELLO) + wgint size; + /* We use fseek rather than stat to determine the file size because + that way we can also verify that the file is readable without + explicitly checking for permissions. Inspired by the POST patch + by Arnaud Wylie. */ + FILE *fp = fopen (filename, "rb"); + if (!fp) + return -1; + fseeko (fp, 0, SEEK_END); + size = ftello (fp); + fclose (fp); + return size; +#else + struct_stat st; + if (stat (filename, &st) < 0) + return -1; + return st.st_size; +#endif +} + +/* stat file names named PREFIX.1, PREFIX.2, etc., until one that + doesn't exist is found. Return a freshly allocated copy of the + unused file name. */ + static char * -unique_name_1 (const char *fileprefix, int count) +unique_name_1 (const char *prefix) { - char *filename; + int count = 1; + int plen = strlen (prefix); + char *template = (char *)alloca (plen + 1 + 24); + char *template_tail = template + plen; - if (count) + memcpy (template, prefix, plen); + *template_tail++ = '.'; + + do + number_to_string (template_tail, count++); + while (file_exists_p (template)); + + return xstrdup (template); +} + +/* Return a unique file name, based on FILE. + + More precisely, if FILE doesn't exist, it is returned unmodified. + If not, FILE.1 is tried, then FILE.2, etc. The first FILE. + file name that doesn't exist is returned. + + The resulting file is not created, only verified that it didn't + exist at the point in time when the function was called. + Therefore, where security matters, don't rely that the file created + by this function exists until you open it with O_EXCL or + equivalent. + + If ALLOW_PASSTHROUGH is 0, it always returns a freshly allocated + string. Otherwise, it may return FILE if the file doesn't exist + (and therefore doesn't need changing). */ + +char * +unique_name (const char *file, int allow_passthrough) +{ + /* If the FILE itself doesn't exist, return it without + modification. */ + if (!file_exists_p (file)) + return allow_passthrough ? (char *)file : xstrdup (file); + + /* Otherwise, find a numeric suffix that results in unused file name + and return it. */ + return unique_name_1 (file); +} + +/* Create a file based on NAME, except without overwriting an existing + file with that name. Providing O_EXCL is correctly implemented, + this function does not have the race condition associated with + opening the file returned by unique_name. */ + +FILE * +unique_create (const char *name, int binary, char **opened_name) +{ + /* unique file name, based on NAME */ + char *uname = unique_name (name, 0); + FILE *fp; + while ((fp = fopen_excl (uname, binary)) == NULL && errno == EEXIST) { - filename = (char *)xmalloc (strlen (fileprefix) + numdigit (count) + 2); - sprintf (filename, "%s.%d", fileprefix, count); + xfree (uname); + uname = unique_name (name, 0); } - else - filename = xstrdup (fileprefix); - - if (!file_exists_p (filename)) - return filename; - else + if (opened_name && fp != NULL) { - xfree (filename); - return NULL; + if (fp) + *opened_name = uname; + else + { + *opened_name = NULL; + xfree (uname); + } } + else + xfree (uname); + return fp; } -/* Return a unique file name, based on PREFIX. */ -char * -unique_name (const char *prefix) +/* Open the file for writing, with the addition that the file is + opened "exclusively". This means that, if the file already exists, + this function will *fail* and errno will be set to EEXIST. If + BINARY is set, the file will be opened in binary mode, equivalent + to fopen's "wb". + + If opening the file fails for any reason, including the file having + previously existed, this function returns NULL and sets errno + appropriately. */ + +FILE * +fopen_excl (const char *fname, int binary) { - char *file = NULL; - int count = 0; - - while (!file) - file = unique_name_1 (prefix, count++); - return file; + int fd; +#ifdef O_EXCL + int flags = O_WRONLY | O_CREAT | O_EXCL; +# ifdef O_BINARY + if (binary) + flags |= O_BINARY; +# endif + fd = open (fname, flags, 0666); + if (fd < 0) + return NULL; + return fdopen (fd, binary ? "wb" : "w"); +#else /* not O_EXCL */ + /* Manually check whether the file exists. This is prone to race + conditions, but systems without O_EXCL haven't deserved + better. */ + if (file_exists_p (fname)) + { + errno = EEXIST; + return NULL; + } + return fopen (fname, binary ? "wb" : "w"); +#endif /* not O_EXCL */ } /* Create DIRECTORY. If some of the pathname components of DIRECTORY @@ -675,8 +579,7 @@ unique_name (const char *prefix) int make_directory (const char *directory) { - int quit = 0; - int i; + int i, ret, quit = 0; char *dir; /* Make a copy of dir, to be able to write to it. Otherwise, the @@ -692,24 +595,27 @@ make_directory (const char *directory) if (!dir[i]) quit = 1; dir[i] = '\0'; - /* Check whether the directory already exists. */ + /* Check whether the directory already exists. Allow creation of + of intermediate directories to fail, as the initial path components + are not necessarily directories! */ if (!file_exists_p (dir)) - { - if (mkdir (dir, 0777) < 0) - return -1; - } + ret = mkdir (dir, 0777); + else + ret = 0; if (quit) break; else dir[i] = '/'; } - return 0; + return ret; } /* Merge BASE with FILE. BASE can be a directory or a file name, FILE - should be a file name. For example, file_merge("/foo/bar", "baz") - will return "/foo/baz". file_merge("/foo/bar/", "baz") will return - "foo/bar/baz". + should be a file name. + + file_merge("/foo/bar", "baz") => "/foo/baz" + file_merge("/foo/bar/", "baz") => "/foo/bar/baz" + file_merge("foo", "bar") => "bar" In other words, it's a simpler and gentler version of uri_merge_1. */ @@ -720,7 +626,7 @@ file_merge (const char *base, const char *file) const char *cut = (const char *)strrchr (base, '/'); if (!cut) - cut = base + strlen (base); + return xstrdup (file); result = (char *)xmalloc (cut - base + 1 + strlen (file) + 1); memcpy (result, base, cut - base); @@ -812,20 +718,37 @@ accdir (const char *directory, enum accd flags) return 1; } -/* Match the end of STRING against PATTERN. For instance: +/* Return non-zero if STRING ends with TAIL. For instance: - match_backwards ("abc", "bc") -> 1 - match_backwards ("abc", "ab") -> 0 - match_backwards ("abc", "abc") -> 1 */ -static int -match_backwards (const char *string, const char *pattern) + match_tail ("abc", "bc", 0) -> 1 + match_tail ("abc", "ab", 0) -> 0 + match_tail ("abc", "abc", 0) -> 1 + + If FOLD_CASE_P is non-zero, the comparison will be + case-insensitive. */ + +int +match_tail (const char *string, const char *tail, int fold_case_p) { int i, j; - for (i = strlen (string), j = strlen (pattern); i >= 0 && j >= 0; i--, j--) - if (string[i] != pattern[j]) - break; - /* If the pattern was exhausted, the match was succesful. */ + /* We want this to be fast, so we code two loops, one with + case-folding, one without. */ + + if (!fold_case_p) + { + for (i = strlen (string), j = strlen (tail); i >= 0 && j >= 0; i--, j--) + if (string[i] != tail[j]) + break; + } + else + { + for (i = strlen (string), j = strlen (tail); i >= 0 && j >= 0; i--, j--) + if (TOLOWER (string[i]) != TOLOWER (tail[j])) + break; + } + + /* If the tail was exhausted, the match was succesful. */ if (j == -1) return 1; else @@ -833,7 +756,7 @@ match_backwards (const char *string, const char *pattern) } /* Checks whether string S matches each element of ACCEPTS. A list - element are matched either with fnmatch() or match_backwards(), + element are matched either with fnmatch() or match_tail(), according to whether the element contains wildcards or not. If the BACKWARD is 0, don't do backward comparison -- just compare @@ -854,7 +777,7 @@ in_acclist (const char *const *accepts, const char *s, int backward) { if (backward) { - if (match_backwards (s, *accepts)) + if (match_tail (s, *accepts, 0)) return 1; } else @@ -867,7 +790,7 @@ in_acclist (const char *const *accepts, const char *s, int backward) return 0; } -/* Return the malloc-ed suffix of STR. For instance: +/* Return the location of STR's suffix (file extension). Examples: suffix ("foo.bar") -> "bar" suffix ("foo.bar.baz") -> "baz" suffix ("/foo/bar") -> NULL @@ -877,18 +800,59 @@ suffix (const char *str) { int i; - for (i = strlen (str); i && str[i] != '/' && str[i] != '.'; i--); + for (i = strlen (str); i && str[i] != '/' && str[i] != '.'; i--) + ; + if (str[i++] == '.') - return xstrdup (str + i); + return (char *)str + i; else return NULL; } -/* Read a line from FP. The function reallocs the storage as needed - to accomodate for any length of the line. Reallocs are done - storage exponentially, doubling the storage after each overflow to - minimize the number of calls to realloc() and fgets(). The newline - character at the end of line is retained. +/* Return non-zero if S contains globbing wildcards (`*', `?', `[' or + `]'). */ + +int +has_wildcards_p (const char *s) +{ + for (; *s; s++) + if (*s == '*' || *s == '?' || *s == '[' || *s == ']') + return 1; + return 0; +} + +/* Return non-zero if FNAME ends with a typical HTML suffix. The + following (case-insensitive) suffixes are presumed to be HTML files: + + html + htm + ?html (`?' matches one character) + + #### CAVEAT. This is not necessarily a good indication that FNAME + refers to a file that contains HTML! */ +int +has_html_suffix_p (const char *fname) +{ + char *suf; + + if ((suf = suffix (fname)) == NULL) + return 0; + if (!strcasecmp (suf, "html")) + return 1; + if (!strcasecmp (suf, "htm")) + return 1; + if (suf[0] && !strcasecmp (suf + 1, "html")) + return 1; + return 0; +} + +/* Read a line from FP and return the pointer to freshly allocated + storage. The storage space is obtained through malloc() and should + be freed with free() when it is no longer needed. + + The length of the line is not limited, except by available memory. + The newline character at the end of line is retained. The line is + terminated with a zero character. After end-of-file is encountered without anything being read, NULL is returned. NULL is also returned on error. To distinguish @@ -898,15 +862,20 @@ char * read_whole_line (FILE *fp) { int length = 0; - int bufsize = 81; + int bufsize = 82; char *line = (char *)xmalloc (bufsize); while (fgets (line + length, bufsize - length, fp)) { length += strlen (line + length); - assert (length > 0); + if (length == 0) + /* Possible for example when reading from a binary file where + a line begins with \0. */ + continue; + if (line[length - 1] == '\n') break; + /* fgets() guarantees to read the whole line, or to use up the space we've given it. We can double the buffer unconditionally. */ @@ -964,11 +933,11 @@ read_file (const char *file) fd = open (file, O_RDONLY); if (fd < 0) return NULL; - fm = xmalloc (sizeof (struct file_memory)); + fm = xnew (struct file_memory); #ifdef HAVE_MMAP { - struct stat buf; + struct_stat buf; if (fstat (fd, &buf) < 0) goto mmap_lose; fm->length = buf.st_size; @@ -1000,7 +969,7 @@ read_file (const char *file) fm->content = xmalloc (size); while (1) { - long nread; + wgint nread; if (fm->length > size / 2) { /* #### I'm not sure whether the whole exponential-growth @@ -1013,7 +982,7 @@ read_file (const char *file) /* Normally, we grow SIZE exponentially to make the number of calls to read() and realloc() logarithmic in relation to file size. However, read() can read an amount of data - smaller than requested, and it would be unreasonably to + smaller than requested, and it would be unreasonable to double SIZE every time *something* was read. Therefore, we double SIZE only when the length exceeds half of the entire allocated size. */ @@ -1111,91 +1080,6 @@ merge_vecs (char **v1, char **v2) xfree (v2); return v1; } - -/* A set of simple-minded routines to store strings in a linked list. - This used to also be used for searching, but now we have hash - tables for that. */ - -/* It's a shame that these simple things like linked lists and hash - tables (see hash.c) need to be implemented over and over again. It - would be nice to be able to use the routines from glib -- see - www.gtk.org for details. However, that would make Wget depend on - glib, and I want to avoid dependencies to external libraries for - reasons of convenience and portability (I suspect Wget is more - portable than anything ever written for Gnome). */ - -/* Append an element to the list. If the list has a huge number of - elements, this can get slow because it has to find the list's - ending. If you think you have to call slist_append in a loop, - think about calling slist_prepend() followed by slist_nreverse(). */ - -slist * -slist_append (slist *l, const char *s) -{ - slist *newel = (slist *)xmalloc (sizeof (slist)); - slist *beg = l; - - newel->string = xstrdup (s); - newel->next = NULL; - - if (!l) - return newel; - /* Find the last element. */ - while (l->next) - l = l->next; - l->next = newel; - return beg; -} - -/* Prepend S to the list. Unlike slist_append(), this is O(1). */ - -slist * -slist_prepend (slist *l, const char *s) -{ - slist *newel = (slist *)xmalloc (sizeof (slist)); - newel->string = xstrdup (s); - newel->next = l; - return newel; -} - -/* Destructively reverse L. */ - -slist * -slist_nreverse (slist *l) -{ - slist *prev = NULL; - while (l) - { - slist *next = l->next; - l->next = prev; - prev = l; - l = next; - } - return prev; -} - -/* Is there a specific entry in the list? */ -int -slist_contains (slist *l, const char *s) -{ - for (; l; l = l->next) - if (!strcmp (l->string, s)) - return 1; - return 0; -} - -/* Free the whole slist. */ -void -slist_free (slist *l) -{ - while (l) - { - slist *n = l->next; - xfree (l->string); - xfree (l); - l = n; - } -} /* Sometimes it's useful to create "sets" of strings, i.e. special hash tables where you want to store strings as keys and merely @@ -1226,6 +1110,22 @@ string_set_contains (struct hash_table *ht, const char *s) return hash_table_contains (ht, s); } +static int +string_set_to_array_mapper (void *key, void *value_ignored, void *arg) +{ + char ***arrayptr = (char ***) arg; + *(*arrayptr)++ = (char *) key; + return 0; +} + +/* Convert the specified string set to array. ARRAY should be large + enough to hold hash_table_count(ht) char pointers. */ + +void string_set_to_array (struct hash_table *ht, char **array) +{ + hash_table_map (ht, string_set_to_array_mapper, &array); +} + static int string_set_free_mapper (void *key, void *value_ignored, void *arg_ignored) { @@ -1257,13 +1157,13 @@ free_keys_and_values (struct hash_table *ht) } -/* Engine for legible and legible_very_long; this function works on - strings. */ +/* Add thousand separators to a number already in string form. Used + by with_thousand_seps and with_thousand_seps_large. */ static char * -legible_1 (const char *repr) +add_thousand_seps (const char *repr) { - static char outbuf[128]; + static char outbuf[48]; int i, i1, mod; char *outptr; const char *inptr; @@ -1271,7 +1171,9 @@ legible_1 (const char *repr) /* Reset the pointers. */ outptr = outbuf; inptr = repr; - /* If the number is negative, shift the pointers. */ + + /* Ignore the sign for the purpose of adding thousand + separators. */ if (*inptr == '-') { *outptr++ = '-'; @@ -1295,170 +1197,260 @@ legible_1 (const char *repr) return outbuf; } -/* Legible -- return a static pointer to the legibly printed long. */ +/* Return a static pointer to the number printed with thousand + separators inserted at the right places. */ + char * -legible (long l) +with_thousand_seps (wgint l) { char inbuf[24]; /* Print the number into the buffer. */ - long_to_string (inbuf, l); - return legible_1 (inbuf); + number_to_string (inbuf, l); + return add_thousand_seps (inbuf); } -/* Write a string representation of NUMBER into the provided buffer. - We cannot use sprintf() because we cannot be sure whether the - platform supports printing of what we chose for VERY_LONG_TYPE. +/* Write a string representation of LARGE_INT NUMBER into the provided + buffer. - Example: Gcc supports `long long' under many platforms, but on many - of those the native libc knows nothing of it and therefore cannot - print it. + It would be dangerous to use sprintf, because the code wouldn't + work on a machine with gcc-provided long long support, but without + libc support for "%lld". However, such old systems platforms + typically lack snprintf and will end up using our version, which + does support "%lld" whereever long longs are available. */ - How long BUFFER needs to be depends on the platform and the content - of NUMBER. For 64-bit VERY_LONG_TYPE (the most common case), 24 - bytes are sufficient. Using more might be a good idea. +static void +large_int_to_string (char *buffer, int bufsize, LARGE_INT number) +{ + snprintf (buffer, bufsize, LARGE_INT_FMT, number); +} - This function does not go through the hoops that long_to_string - goes to because it doesn't aspire to be fast. (It's called perhaps - once in a Wget run.) */ +/* The same as with_thousand_seps, but works on LARGE_INT. */ -static void -very_long_to_string (char *buffer, VERY_LONG_TYPE number) +char * +with_thousand_seps_large (LARGE_INT l) { - int i = 0; - int j; + char inbuf[48]; + large_int_to_string (inbuf, sizeof (inbuf), l); + return add_thousand_seps (inbuf); +} - /* Print the number backwards... */ - do +/* N, a byte quantity, is converted to a human-readable abberviated + form a la sizes printed by `ls -lh'. The result is written to a + static buffer, a pointer to which is returned. + + Unlike `with_thousand_seps', this approximates to the nearest unit. + Quoting GNU libit: "Most people visually process strings of 3-4 + digits effectively, but longer strings of digits are more prone to + misinterpretation. Hence, converting to an abbreviated form + usually improves readability." + + This intentionally uses kilobyte (KB), megabyte (MB), etc. in their + original computer science meaning of "multiples of 1024". + Multiples of 1000 would be useless since Wget already adds thousand + separators for legibility. We don't use the "*bibyte" names + invented in 1998, and seldom used in practice. Wikipedia's entry + on kilobyte discusses this in some detail. */ + +char * +human_readable (wgint n) +{ + /* These suffixes are compatible with those of GNU `ls -lh'. */ + static char powers[] = { - buffer[i++] = '0' + number % 10; - number /= 10; + 'K', /* kilobyte, 2^10 bytes */ + 'M', /* megabyte, 2^20 bytes */ + 'G', /* gigabyte, 2^30 bytes */ + 'T', /* terabyte, 2^40 bytes */ + 'P', /* petabyte, 2^50 bytes */ + 'E', /* exabyte, 2^60 bytes */ + }; + static char buf[8]; + int i; + + /* If the quantity is smaller than 1K, just print it. */ + if (n < 1024) + { + snprintf (buf, sizeof (buf), "%d", (int) n); + return buf; } - while (number); - /* ...and reverse the order of the digits. */ - for (j = 0; j < i / 2; j++) + /* Loop over powers, dividing N with 1024 in each iteration. This + works unchanged for all sizes of wgint, while still avoiding + non-portable `long double' arithmetic. */ + for (i = 0; i < countof (powers); i++) { - char c = buffer[j]; - buffer[j] = buffer[i - 1 - j]; - buffer[i - 1 - j] = c; + /* At each iteration N is greater than the *subsequent* power. + That way N/1024.0 produces a decimal number in the units of + *this* power. */ + if ((n >> 10) < 1024 || i == countof (powers) - 1) + { + /* Must cast to long first because MS VC can't directly cast + __int64 to double. (This is safe because N is known to + be <2**20.) */ + double val = (double) (long) n / 1024.0; + /* Print values smaller than 10 with one decimal digits, and + others without any decimals. */ + snprintf (buf, sizeof (buf), "%.*f%c", + val < 10 ? 1 : 0, val, powers[i]); + return buf; + } + n >>= 10; } - buffer[i] = '\0'; + return NULL; /* unreached */ } -/* The same as legible(), but works on VERY_LONG_TYPE. See sysdep.h. */ -char * -legible_very_long (VERY_LONG_TYPE l) -{ - char inbuf[128]; - /* Print the number into the buffer. */ - very_long_to_string (inbuf, l); - return legible_1 (inbuf); -} +/* Count the digits in the provided number. Used to allocate space + when printing numbers. */ -/* Count the digits in a (long) integer. */ int -numdigit (long a) +numdigit (wgint number) { - int res = 1; - if (a < 0) - { - a = -a; - ++res; - } - while ((a /= 10) != 0) - ++res; - return res; + int cnt = 1; + if (number < 0) + ++cnt; /* accomodate '-' */ + while ((number /= 10) != 0) + ++cnt; + return cnt; } -#define ONE_DIGIT(figure) *p++ = n / (figure) + '0' -#define ONE_DIGIT_ADVANCE(figure) (ONE_DIGIT (figure), n %= (figure)) - -#define DIGITS_1(figure) ONE_DIGIT (figure) -#define DIGITS_2(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_1 ((figure) / 10) -#define DIGITS_3(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_2 ((figure) / 10) -#define DIGITS_4(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_3 ((figure) / 10) -#define DIGITS_5(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_4 ((figure) / 10) -#define DIGITS_6(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_5 ((figure) / 10) -#define DIGITS_7(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_6 ((figure) / 10) -#define DIGITS_8(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_7 ((figure) / 10) -#define DIGITS_9(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_8 ((figure) / 10) -#define DIGITS_10(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_9 ((figure) / 10) - -/* DIGITS_<11-20> are only used on machines with 64-bit longs. */ - -#define DIGITS_11(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_10 ((figure) / 10) -#define DIGITS_12(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_11 ((figure) / 10) -#define DIGITS_13(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_12 ((figure) / 10) -#define DIGITS_14(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_13 ((figure) / 10) -#define DIGITS_15(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_14 ((figure) / 10) -#define DIGITS_16(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_15 ((figure) / 10) -#define DIGITS_17(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_16 ((figure) / 10) -#define DIGITS_18(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_17 ((figure) / 10) -#define DIGITS_19(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_18 ((figure) / 10) - -/* Print NUMBER to BUFFER in base 10. This is completely equivalent - to `sprintf(buffer, "%ld", number)', only much faster. +#define PR(mask) *p++ = n / (mask) + '0' + +/* DIGITS_ is used to print a D-digit number and should be called + with mask==10^(D-1). It prints n/mask (the first digit), reducing + n to n%mask (the remaining digits), and calling DIGITS_. + Recursively this continues until DIGITS_1 is invoked. */ + +#define DIGITS_1(mask) PR (mask) +#define DIGITS_2(mask) PR (mask), n %= (mask), DIGITS_1 ((mask) / 10) +#define DIGITS_3(mask) PR (mask), n %= (mask), DIGITS_2 ((mask) / 10) +#define DIGITS_4(mask) PR (mask), n %= (mask), DIGITS_3 ((mask) / 10) +#define DIGITS_5(mask) PR (mask), n %= (mask), DIGITS_4 ((mask) / 10) +#define DIGITS_6(mask) PR (mask), n %= (mask), DIGITS_5 ((mask) / 10) +#define DIGITS_7(mask) PR (mask), n %= (mask), DIGITS_6 ((mask) / 10) +#define DIGITS_8(mask) PR (mask), n %= (mask), DIGITS_7 ((mask) / 10) +#define DIGITS_9(mask) PR (mask), n %= (mask), DIGITS_8 ((mask) / 10) +#define DIGITS_10(mask) PR (mask), n %= (mask), DIGITS_9 ((mask) / 10) + +/* DIGITS_<11-20> are only used on machines with 64-bit wgints. */ + +#define DIGITS_11(mask) PR (mask), n %= (mask), DIGITS_10 ((mask) / 10) +#define DIGITS_12(mask) PR (mask), n %= (mask), DIGITS_11 ((mask) / 10) +#define DIGITS_13(mask) PR (mask), n %= (mask), DIGITS_12 ((mask) / 10) +#define DIGITS_14(mask) PR (mask), n %= (mask), DIGITS_13 ((mask) / 10) +#define DIGITS_15(mask) PR (mask), n %= (mask), DIGITS_14 ((mask) / 10) +#define DIGITS_16(mask) PR (mask), n %= (mask), DIGITS_15 ((mask) / 10) +#define DIGITS_17(mask) PR (mask), n %= (mask), DIGITS_16 ((mask) / 10) +#define DIGITS_18(mask) PR (mask), n %= (mask), DIGITS_17 ((mask) / 10) +#define DIGITS_19(mask) PR (mask), n %= (mask), DIGITS_18 ((mask) / 10) + +/* SPRINTF_WGINT is used by number_to_string to handle pathological + cases and to portably support strange sizes of wgint. Ideally this + would just use "%j" and intmax_t, but many systems don't support + it, so it's used only if nothing else works. */ +#if SIZEOF_LONG >= SIZEOF_WGINT +# define SPRINTF_WGINT(buf, n) sprintf (buf, "%ld", (long) (n)) +#else +# if SIZEOF_LONG_LONG >= SIZEOF_WGINT +# define SPRINTF_WGINT(buf, n) sprintf (buf, "%lld", (long long) (n)) +# else +# ifdef WINDOWS +# define SPRINTF_WGINT(buf, n) sprintf (buf, "%I64", (__int64) (n)) +# else +# define SPRINTF_WGINT(buf, n) sprintf (buf, "%j", (intmax_t) (n)) +# endif +# endif +#endif + +/* Shorthand for casting to wgint. */ +#define W wgint + +/* Print NUMBER to BUFFER in base 10. This is equivalent to + `sprintf(buffer, "%lld", (long long) number)', only typically much + faster and portable to machines without long long. The speedup may make a difference in programs that frequently convert numbers to strings. Some implementations of sprintf, particularly the one in GNU libc, have been known to be extremely - slow compared to this function. + slow when converting integers to strings. - BUFFER should accept as many bytes as you expect the number to take - up. On machines with 64-bit longs the maximum needed size is 24 - bytes. That includes the worst-case digits, the optional `-' sign, - and the trailing \0. */ + Return the pointer to the location where the terminating zero was + printed. (Equivalent to calling buffer+strlen(buffer) after the + function is done.) -void -long_to_string (char *buffer, long number) + BUFFER should be big enough to accept as many bytes as you expect + the number to take up. On machines with 64-bit longs the maximum + needed size is 24 bytes. That includes the digits needed for the + largest 64-bit number, the `-' sign in case it's negative, and the + terminating '\0'. */ + +char * +number_to_string (char *buffer, wgint number) { char *p = buffer; - long n = number; + wgint n = number; -#if (SIZEOF_LONG != 4) && (SIZEOF_LONG != 8) +#if (SIZEOF_WGINT != 4) && (SIZEOF_WGINT != 8) /* We are running in a strange or misconfigured environment. Let sprintf cope with it. */ - sprintf (buffer, "%ld", n); -#else /* (SIZEOF_LONG == 4) || (SIZEOF_LONG == 8) */ + SPRINTF_WGINT (buffer, n); + p += strlen (buffer); +#else /* (SIZEOF_WGINT == 4) || (SIZEOF_WGINT == 8) */ if (n < 0) { + if (n < -WGINT_MAX) + { + /* -n would overflow. Have sprintf deal with this. */ + SPRINTF_WGINT (buffer, n); + p += strlen (buffer); + return p; + } + *p++ = '-'; n = -n; } - if (n < 10) { DIGITS_1 (1); } - else if (n < 100) { DIGITS_2 (10); } - else if (n < 1000) { DIGITS_3 (100); } - else if (n < 10000) { DIGITS_4 (1000); } - else if (n < 100000) { DIGITS_5 (10000); } - else if (n < 1000000) { DIGITS_6 (100000); } - else if (n < 10000000) { DIGITS_7 (1000000); } - else if (n < 100000000) { DIGITS_8 (10000000); } - else if (n < 1000000000) { DIGITS_9 (100000000); } -#if SIZEOF_LONG == 4 - /* ``if (1)'' serves only to preserve editor indentation. */ - else if (1) { DIGITS_10 (1000000000); } -#else /* SIZEOF_LONG != 4 */ - else if (n < 10000000000L) { DIGITS_10 (1000000000L); } - else if (n < 100000000000L) { DIGITS_11 (10000000000L); } - else if (n < 1000000000000L) { DIGITS_12 (100000000000L); } - else if (n < 10000000000000L) { DIGITS_13 (1000000000000L); } - else if (n < 100000000000000L) { DIGITS_14 (10000000000000L); } - else if (n < 1000000000000000L) { DIGITS_15 (100000000000000L); } - else if (n < 10000000000000000L) { DIGITS_16 (1000000000000000L); } - else if (n < 100000000000000000L) { DIGITS_17 (10000000000000000L); } - else if (n < 1000000000000000000L) { DIGITS_18 (100000000000000000L); } - else { DIGITS_19 (1000000000000000000L); } -#endif /* SIZEOF_LONG != 4 */ + /* Use the DIGITS_ macro appropriate for N's number of digits. That + way printing any N is fully open-coded without a loop or jump. + (Also see description of DIGITS_*.) */ + + if (n < 10) DIGITS_1 (1); + else if (n < 100) DIGITS_2 (10); + else if (n < 1000) DIGITS_3 (100); + else if (n < 10000) DIGITS_4 (1000); + else if (n < 100000) DIGITS_5 (10000); + else if (n < 1000000) DIGITS_6 (100000); + else if (n < 10000000) DIGITS_7 (1000000); + else if (n < 100000000) DIGITS_8 (10000000); + else if (n < 1000000000) DIGITS_9 (100000000); +#if SIZEOF_WGINT == 4 + /* wgint is 32 bits wide: no number has more than 10 digits. */ + else DIGITS_10 (1000000000); +#else + /* wgint is 64 bits wide: handle numbers with more than 9 decimal + digits. Constants are constructed by compile-time multiplication + to avoid dealing with different notations for 64-bit constants + (nnnL, nnnLL, and nnnI64, depending on the compiler). */ + else if (n < 10*(W)1000000000) DIGITS_10 (1000000000); + else if (n < 100*(W)1000000000) DIGITS_11 (10*(W)1000000000); + else if (n < 1000*(W)1000000000) DIGITS_12 (100*(W)1000000000); + else if (n < 10000*(W)1000000000) DIGITS_13 (1000*(W)1000000000); + else if (n < 100000*(W)1000000000) DIGITS_14 (10000*(W)1000000000); + else if (n < 1000000*(W)1000000000) DIGITS_15 (100000*(W)1000000000); + else if (n < 10000000*(W)1000000000) DIGITS_16 (1000000*(W)1000000000); + else if (n < 100000000*(W)1000000000) DIGITS_17 (10000000*(W)1000000000); + else if (n < 1000000000*(W)1000000000) DIGITS_18 (100000000*(W)1000000000); + else DIGITS_19 (1000000000*(W)1000000000); +#endif *p = '\0'; -#endif /* (SIZEOF_LONG == 4) || (SIZEOF_LONG == 8) */ -} +#endif /* (SIZEOF_WGINT == 4) || (SIZEOF_WGINT == 8) */ -#undef ONE_DIGIT -#undef ONE_DIGIT_ADVANCE + return p; +} +#undef PR +#undef W #undef DIGITS_1 #undef DIGITS_2 #undef DIGITS_3 @@ -1478,257 +1470,497 @@ long_to_string (char *buffer, long number) #undef DIGITS_17 #undef DIGITS_18 #undef DIGITS_19 + +#define RING_SIZE 3 + +/* Print NUMBER to a statically allocated string and return a pointer + to the printed representation. + + This function is intended to be used in conjunction with printf. + It is hard to portably print wgint values: + a) you cannot use printf("%ld", number) because wgint can be long + long on 32-bit machines with LFS. + b) you cannot use printf("%lld", number) because NUMBER could be + long on 32-bit machines without LFS, or on 64-bit machines, + which do not require LFS. Also, Windows doesn't support %lld. + c) you cannot use printf("%j", (int_max_t) number) because not all + versions of printf support "%j", the most notable being the one + on Windows. + d) you cannot #define WGINT_FMT to the appropriate format and use + printf(WGINT_FMT, number) because that would break translations + for user-visible messages, such as printf("Downloaded: %d + bytes\n", number). + + What you should use instead is printf("%s", number_to_static_string + (number)). + + CAVEAT: since the function returns pointers to static data, you + must be careful to copy its result before calling it again. + However, to make it more useful with printf, the function maintains + an internal ring of static buffers to return. That way things like + printf("%s %s", number_to_static_string (num1), + number_to_static_string (num2)) work as expected. Three buffers + are currently used, which means that "%s %s %s" will work, but "%s + %s %s %s" won't. If you need to print more than three wgints, + bump the RING_SIZE (or rethink your message.) */ + +char * +number_to_static_string (wgint number) +{ + static char ring[RING_SIZE][24]; + static int ringpos; + char *buf = ring[ringpos]; + number_to_string (buf, number); + ringpos = (ringpos + 1) % RING_SIZE; + return buf; +} -/* Support for timers. */ - -#undef TIMER_WINDOWS -#undef TIMER_GETTIMEOFDAY -#undef TIMER_TIME - -/* Depending on the OS and availability of gettimeofday(), one and - only one of the above constants will be defined. Virtually all - modern Unix systems will define TIMER_GETTIMEOFDAY; Windows will - use TIMER_WINDOWS. TIMER_TIME is a catch-all method for - non-Windows systems without gettimeofday. - - #### Perhaps we should also support ftime(), which exists on old - BSD 4.2-influenced systems? (It also existed under MS DOS Borland - C, if memory serves me.) */ - -#ifdef WINDOWS -# define TIMER_WINDOWS -#else /* not WINDOWS */ -# ifdef HAVE_GETTIMEOFDAY -# define TIMER_GETTIMEOFDAY -# else -# define TIMER_TIME -# endif -#endif /* not WINDOWS */ +/* Determine the width of the terminal we're running on. If that's + not possible, return 0. */ -struct wget_timer { -#ifdef TIMER_GETTIMEOFDAY - long secs; - long usecs; -#endif +int +determine_screen_width (void) +{ + /* If there's a way to get the terminal size using POSIX + tcgetattr(), somebody please tell me. */ +#ifdef TIOCGWINSZ + int fd; + struct winsize wsz; -#ifdef TIMER_TIME - time_t secs; -#endif + if (opt.lfilename != NULL) + return 0; + + fd = fileno (stderr); + if (ioctl (fd, TIOCGWINSZ, &wsz) < 0) + return 0; /* most likely ENOTTY */ + + return wsz.ws_col; +#else /* not TIOCGWINSZ */ +# ifdef WINDOWS + CONSOLE_SCREEN_BUFFER_INFO csbi; + if (!GetConsoleScreenBufferInfo (GetStdHandle (STD_ERROR_HANDLE), &csbi)) + return 0; + return csbi.dwSize.X; +# else /* neither WINDOWS nor TIOCGWINSZ */ + return 0; +#endif /* neither WINDOWS nor TIOCGWINSZ */ +#endif /* not TIOCGWINSZ */ +} + +/* Return a random number between 0 and MAX-1, inclusive. + + If MAX is greater than the value of RAND_MAX+1 on the system, the + returned value will be in the range [0, RAND_MAX]. This may be + fixed in a future release. + + The random number generator is seeded automatically the first time + it is called. -#ifdef TIMER_WINDOWS - ULARGE_INTEGER wintime; + This uses rand() for portability. It has been suggested that + random() offers better randomness, but this is not required for + Wget, so I chose to go for simplicity and use rand + unconditionally. + + DO NOT use this for cryptographic purposes. It is only meant to be + used in situations where quality of the random numbers returned + doesn't really matter. */ + +int +random_number (int max) +{ + static int seeded; + double bounded; + int rnd; + + if (!seeded) + { + srand (time (NULL)); + seeded = 1; + } + rnd = rand (); + + /* On systems that don't define RAND_MAX, assume it to be 2**15 - 1, + and enforce that assumption by masking other bits. */ +#ifndef RAND_MAX +# define RAND_MAX 32767 + rnd &= RAND_MAX; #endif -}; -/* Allocate a timer. It is not legal to do anything with a freshly - allocated timer, except call wtimer_reset() or wtimer_delete(). */ + /* This is equivalent to rand() % max, but uses the high-order bits + for better randomness on architecture where rand() is implemented + using a simple congruential generator. */ + + bounded = (double)max * rnd / (RAND_MAX + 1.0); + return (int)bounded; +} + +/* Return a random uniformly distributed floating point number in the + [0, 1) range. The precision of returned numbers is 9 digits. -struct wget_timer * -wtimer_allocate (void) + Modify this to use erand48() where available! */ + +double +random_float (void) { - struct wget_timer *wt = - (struct wget_timer *)xmalloc (sizeof (struct wget_timer)); - return wt; + /* We can't rely on any specific value of RAND_MAX, but I'm pretty + sure it's greater than 1000. */ + int rnd1 = random_number (1000); + int rnd2 = random_number (1000); + int rnd3 = random_number (1000); + return rnd1 / 1000.0 + rnd2 / 1000000.0 + rnd3 / 1000000000.0; } + +/* Implementation of run_with_timeout, a generic timeout-forcing + routine for systems with Unix-like signal handling. */ + +#ifdef USE_SIGNAL_TIMEOUT +# ifdef HAVE_SIGSETJMP +# define SETJMP(env) sigsetjmp (env, 1) -/* Allocate a new timer and reset it. Return the new timer. */ +static sigjmp_buf run_with_timeout_env; -struct wget_timer * -wtimer_new (void) +static RETSIGTYPE +abort_run_with_timeout (int sig) { - struct wget_timer *wt = wtimer_allocate (); - wtimer_reset (wt); - return wt; + assert (sig == SIGALRM); + siglongjmp (run_with_timeout_env, -1); } +# else /* not HAVE_SIGSETJMP */ +# define SETJMP(env) setjmp (env) -/* Free the resources associated with the timer. Its further use is - prohibited. */ +static jmp_buf run_with_timeout_env; -void -wtimer_delete (struct wget_timer *wt) +static RETSIGTYPE +abort_run_with_timeout (int sig) { - xfree (wt); + assert (sig == SIGALRM); + /* We don't have siglongjmp to preserve the set of blocked signals; + if we longjumped out of the handler at this point, SIGALRM would + remain blocked. We must unblock it manually. */ + int mask = siggetmask (); + mask &= ~sigmask (SIGALRM); + sigsetmask (mask); + + /* Now it's safe to longjump. */ + longjmp (run_with_timeout_env, -1); } +# endif /* not HAVE_SIGSETJMP */ -/* Reset timer WT. This establishes the starting point from which - wtimer_elapsed() will return the number of elapsed - milliseconds. It is allowed to reset a previously used timer. */ +/* Arrange for SIGALRM to be delivered in TIMEOUT seconds. This uses + setitimer where available, alarm otherwise. -void -wtimer_reset (struct wget_timer *wt) + TIMEOUT should be non-zero. If the timeout value is so small that + it would be rounded to zero, it is rounded to the least legal value + instead (1us for setitimer, 1s for alarm). That ensures that + SIGALRM will be delivered in all cases. */ + +static void +alarm_set (double timeout) { -#ifdef TIMER_GETTIMEOFDAY - struct timeval t; - gettimeofday (&t, NULL); - wt->secs = t.tv_sec; - wt->usecs = t.tv_usec; -#endif +#ifdef ITIMER_REAL + /* Use the modern itimer interface. */ + struct itimerval itv; + xzero (itv); + itv.it_value.tv_sec = (long) timeout; + itv.it_value.tv_usec = 1000000 * (timeout - (long)timeout); + if (itv.it_value.tv_sec == 0 && itv.it_value.tv_usec == 0) + /* Ensure that we wait for at least the minimum interval. + Specifying zero would mean "wait forever". */ + itv.it_value.tv_usec = 1; + setitimer (ITIMER_REAL, &itv, NULL); +#else /* not ITIMER_REAL */ + /* Use the old alarm() interface. */ + int secs = (int) timeout; + if (secs == 0) + /* Round TIMEOUTs smaller than 1 to 1, not to zero. This is + because alarm(0) means "never deliver the alarm", i.e. "wait + forever", which is not what someone who specifies a 0.5s + timeout would expect. */ + secs = 1; + alarm (secs); +#endif /* not ITIMER_REAL */ +} -#ifdef TIMER_TIME - wt->secs = time (NULL); -#endif +/* Cancel the alarm set with alarm_set. */ -#ifdef TIMER_WINDOWS - FILETIME ft; - SYSTEMTIME st; - GetSystemTime (&st); - SystemTimeToFileTime (&st, &ft); - wt->wintime.HighPart = ft.dwHighDateTime; - wt->wintime.LowPart = ft.dwLowDateTime; -#endif +static void +alarm_cancel (void) +{ +#ifdef ITIMER_REAL + struct itimerval disable; + xzero (disable); + setitimer (ITIMER_REAL, &disable, NULL); +#else /* not ITIMER_REAL */ + alarm (0); +#endif /* not ITIMER_REAL */ } -/* Return the number of milliseconds elapsed since the timer was last - reset. It is allowed to call this function more than once to get - increasingly higher elapsed values. */ +/* Call FUN(ARG), but don't allow it to run for more than TIMEOUT + seconds. Returns non-zero if the function was interrupted with a + timeout, zero otherwise. + + This works by setting up SIGALRM to be delivered in TIMEOUT seconds + using setitimer() or alarm(). The timeout is enforced by + longjumping out of the SIGALRM handler. This has several + advantages compared to the traditional approach of relying on + signals causing system calls to exit with EINTR: + + * The callback function is *forcibly* interrupted after the + timeout expires, (almost) regardless of what it was doing and + whether it was in a syscall. For example, a calculation that + takes a long time is interrupted as reliably as an IO + operation. + + * It works with both SYSV and BSD signals because it doesn't + depend on the default setting of SA_RESTART. + + * It doesn't require special handler setup beyond a simple call + to signal(). (It does use sigsetjmp/siglongjmp, but they're + optional.) -long -wtimer_elapsed (struct wget_timer *wt) + The only downside is that, if FUN allocates internal resources that + are normally freed prior to exit from the functions, they will be + lost in case of timeout. */ + +int +run_with_timeout (double timeout, void (*fun) (void *), void *arg) { -#ifdef TIMER_GETTIMEOFDAY - struct timeval t; - gettimeofday (&t, NULL); - return (t.tv_sec - wt->secs) * 1000 + (t.tv_usec - wt->usecs) / 1000; -#endif + int saved_errno; -#ifdef TIMER_TIME - time_t now = time (NULL); - return 1000 * (now - wt->secs); -#endif + if (timeout == 0) + { + fun (arg); + return 0; + } -#ifdef WINDOWS - FILETIME ft; - SYSTEMTIME st; - ULARGE_INTEGER uli; - GetSystemTime (&st); - SystemTimeToFileTime (&st, &ft); - uli.HighPart = ft.dwHighDateTime; - uli.LowPart = ft.dwLowDateTime; - return (long)((uli.QuadPart - wt->wintime.QuadPart) / 10000); -#endif + signal (SIGALRM, abort_run_with_timeout); + if (SETJMP (run_with_timeout_env) != 0) + { + /* Longjumped out of FUN with a timeout. */ + signal (SIGALRM, SIG_DFL); + return 1; + } + alarm_set (timeout); + fun (arg); + + /* Preserve errno in case alarm() or signal() modifies it. */ + saved_errno = errno; + alarm_cancel (); + signal (SIGALRM, SIG_DFL); + errno = saved_errno; + + return 0; } -/* Return the assessed granularity of the timer implementation. This - is important for certain code that tries to deal with "zero" time - intervals. */ +#else /* not USE_SIGNAL_TIMEOUT */ + +#ifndef WINDOWS +/* A stub version of run_with_timeout that just calls FUN(ARG). Don't + define it under Windows, because Windows has its own version of + run_with_timeout that uses threads. */ -long -wtimer_granularity (void) +int +run_with_timeout (double timeout, void (*fun) (void *), void *arg) { -#ifdef TIMER_GETTIMEOFDAY - /* Granularity of gettimeofday is hugely architecture-dependent. - However, it appears that on modern machines it is better than - 1ms. */ - return 1; -#endif + fun (arg); + return 0; +} +#endif /* not WINDOWS */ +#endif /* not USE_SIGNAL_TIMEOUT */ + +#ifndef WINDOWS -#ifdef TIMER_TIME - /* This is clear. */ - return 1000; -#endif +/* Sleep the specified amount of seconds. On machines without + nanosleep(), this may sleep shorter if interrupted by signals. */ -#ifdef TIMER_WINDOWS - /* ? */ - return 1; -#endif +void +xsleep (double seconds) +{ +#ifdef HAVE_NANOSLEEP + /* nanosleep is the preferred interface because it offers high + accuracy and, more importantly, because it allows us to reliably + restart receiving a signal such as SIGWINCH. (There was an + actual Debian bug report about --limit-rate malfunctioning while + the terminal was being resized.) */ + struct timespec sleep, remaining; + sleep.tv_sec = (long) seconds; + sleep.tv_nsec = 1000000000 * (seconds - (long) seconds); + while (nanosleep (&sleep, &remaining) < 0 && errno == EINTR) + /* If nanosleep has been interrupted by a signal, adjust the + sleeping period and return to sleep. */ + sleep = remaining; +#else /* not HAVE_NANOSLEEP */ +#ifdef HAVE_USLEEP + /* If usleep is available, use it in preference to select. */ + if (seconds >= 1) + { + /* On some systems, usleep cannot handle values larger than + 1,000,000. If the period is larger than that, use sleep + first, then add usleep for subsecond accuracy. */ + sleep (seconds); + seconds -= (long) seconds; + } + usleep (seconds * 1000000); +#else /* not HAVE_USLEEP */ +#ifdef HAVE_SELECT + /* Note that, although Windows supports select, this sleeping + strategy doesn't work there because Winsock's select doesn't + implement timeout when it is passed NULL pointers for all fd + sets. (But it does work under Cygwin, which implements its own + select.) */ + struct timeval sleep; + sleep.tv_sec = (long) seconds; + sleep.tv_usec = 1000000 * (seconds - (long) seconds); + select (0, NULL, NULL, NULL, &sleep); + /* If select returns -1 and errno is EINTR, it means we were + interrupted by a signal. But without knowing how long we've + actually slept, we can't return to sleep. Using gettimeofday to + track sleeps is slow and unreliable due to clock skew. */ +#else /* not HAVE_SELECT */ + sleep (seconds); +#endif /* not HAVE_SELECT */ +#endif /* not HAVE_USLEEP */ +#endif /* not HAVE_NANOSLEEP */ } - -/* This should probably be at a better place, but it doesn't really - fit into html-parse.c. */ -/* The function returns the pointer to the malloc-ed quoted version of - string s. It will recognize and quote numeric and special graphic - entities, as per RFC1866: +#endif /* not WINDOWS */ - `&' -> `&' - `<' -> `<' - `>' -> `>' - `"' -> `"' - SP -> ` ' +/* Encode the string S of length LENGTH to base64 format and place it + to STORE. STORE will be 0-terminated, and must point to a writable + buffer of at least 1+BASE64_LENGTH(length) bytes. */ - No other entities are recognized or replaced. */ -char * -html_quote_string (const char *s) +void +base64_encode (const char *s, char *store, int length) { - const char *b = s; - char *p, *res; + /* Conversion table. */ + static char tbl[64] = { + 'A','B','C','D','E','F','G','H', + 'I','J','K','L','M','N','O','P', + 'Q','R','S','T','U','V','W','X', + 'Y','Z','a','b','c','d','e','f', + 'g','h','i','j','k','l','m','n', + 'o','p','q','r','s','t','u','v', + 'w','x','y','z','0','1','2','3', + '4','5','6','7','8','9','+','/' + }; int i; + unsigned char *p = (unsigned char *)store; - /* Pass through the string, and count the new size. */ - for (i = 0; *s; s++, i++) - { - if (*s == '&') - i += 4; /* `amp;' */ - else if (*s == '<' || *s == '>') - i += 3; /* `lt;' and `gt;' */ - else if (*s == '\"') - i += 5; /* `quot;' */ - else if (*s == ' ') - i += 4; /* #32; */ - } - res = (char *)xmalloc (i + 1); - s = b; - for (p = res; *s; s++) + /* Transform the 3x8 bits to 4x6 bits, as required by base64. */ + for (i = 0; i < length; i += 3) { - switch (*s) - { - case '&': - *p++ = '&'; - *p++ = 'a'; - *p++ = 'm'; - *p++ = 'p'; - *p++ = ';'; - break; - case '<': case '>': - *p++ = '&'; - *p++ = (*s == '<' ? 'l' : 'g'); - *p++ = 't'; - *p++ = ';'; - break; - case '\"': - *p++ = '&'; - *p++ = 'q'; - *p++ = 'u'; - *p++ = 'o'; - *p++ = 't'; - *p++ = ';'; - break; - case ' ': - *p++ = '&'; - *p++ = '#'; - *p++ = '3'; - *p++ = '2'; - *p++ = ';'; - break; - default: - *p++ = *s; - } + *p++ = tbl[s[0] >> 2]; + *p++ = tbl[((s[0] & 3) << 4) + (s[1] >> 4)]; + *p++ = tbl[((s[1] & 0xf) << 2) + (s[2] >> 6)]; + *p++ = tbl[s[2] & 0x3f]; + s += 3; } + /* Pad the result if necessary... */ + if (i == length + 1) + *(p - 1) = '='; + else if (i == length + 2) + *(p - 1) = *(p - 2) = '='; + /* ...and zero-terminate it. */ *p = '\0'; - return res; } -/* Determine the width of the terminal we're running on. If that's - not possible, return 0. */ +#define IS_ASCII(c) (((c) & 0x80) == 0) +#define IS_BASE64(c) ((IS_ASCII (c) && base64_char_to_value[c] >= 0) || c == '=') + +/* Get next character from the string, except that non-base64 + characters are ignored, as mandated by rfc2045. */ +#define NEXT_BASE64_CHAR(c, p) do { \ + c = *p++; \ +} while (c != '\0' && !IS_BASE64 (c)) + +/* Decode data from BASE64 (assumed to be encoded as base64) into + memory pointed to by TO. TO should be large enough to accomodate + the decoded data, which is guaranteed to be less than + strlen(base64). + + Since TO is assumed to contain binary data, it is not + NUL-terminated. The function returns the length of the data + written to TO. -1 is returned in case of error caused by malformed + base64 input. */ int -determine_screen_width (void) +base64_decode (const char *base64, char *to) { - /* If there's a way to get the terminal size using POSIX - tcgetattr(), somebody please tell me. */ -#ifndef TIOCGWINSZ - return 0; -#else /* TIOCGWINSZ */ - int fd; - struct winsize wsz; + /* Table of base64 values for first 128 characters. */ + static short base64_char_to_value[128] = + { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 0- 9 */ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 10- 19 */ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 20- 29 */ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 30- 39 */ + -1, -1, -1, 62, -1, -1, -1, 63, 52, 53, /* 40- 49 */ + 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, /* 50- 59 */ + -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, /* 60- 69 */ + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, /* 70- 79 */ + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, /* 80- 89 */ + 25, -1, -1, -1, -1, -1, -1, 26, 27, 28, /* 90- 99 */ + 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, /* 100-109 */ + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, /* 110-119 */ + 49, 50, 51, -1, -1, -1, -1, -1 /* 120-127 */ + }; + + const char *p = base64; + char *q = to; - if (opt.lfilename != NULL) - return 0; + while (1) + { + unsigned char c; + unsigned long value; - fd = fileno (stderr); - if (ioctl (fd, TIOCGWINSZ, &wsz) < 0) - return 0; /* most likely ENOTTY */ + /* Process first byte of a quadruplet. */ + NEXT_BASE64_CHAR (c, p); + if (!c) + break; + if (c == '=') + return -1; /* illegal '=' while decoding base64 */ + value = base64_char_to_value[c] << 18; + + /* Process scond byte of a quadruplet. */ + NEXT_BASE64_CHAR (c, p); + if (!c) + return -1; /* premature EOF while decoding base64 */ + if (c == '=') + return -1; /* illegal `=' while decoding base64 */ + value |= base64_char_to_value[c] << 12; + *q++ = value >> 16; + + /* Process third byte of a quadruplet. */ + NEXT_BASE64_CHAR (c, p); + if (!c) + return -1; /* premature EOF while decoding base64 */ + + if (c == '=') + { + NEXT_BASE64_CHAR (c, p); + if (!c) + return -1; /* premature EOF while decoding base64 */ + if (c != '=') + return -1; /* padding `=' expected but not found */ + continue; + } - return wsz.ws_col; -#endif /* TIOCGWINSZ */ + value |= base64_char_to_value[c] << 6; + *q++ = 0xff & value >> 8; + + /* Process fourth byte of a quadruplet. */ + NEXT_BASE64_CHAR (c, p); + if (!c) + return -1; /* premature EOF while decoding base64 */ + if (c == '=') + continue; + + value |= base64_char_to_value[c]; + *q++ = 0xff & value; + } + + return q - to; } + +#undef IS_ASCII +#undef IS_BASE64 +#undef NEXT_BASE64_CHAR