X-Git-Url: http://sjero.net/git/?a=blobdiff_plain;f=src%2Futils.c;h=a427e7367706d1c043844e749f85f26c9fe2059a;hb=5f0a2b3f0846dd4c2f72fc62e7171200d1fd6e06;hp=77ea43489df4574ef11bf59cf838adf4fed66f62;hpb=dfc1eb576672755588d24cca9c901f0934559914;p=wget diff --git a/src/utils.c b/src/utils.c index 77ea4348..a427e736 100644 --- a/src/utils.c +++ b/src/utils.c @@ -1,21 +1,31 @@ -/* Various functions of utilitarian nature. - Copyright (C) 1995, 1996, 1997, 1998, 2000 Free Software Foundation, Inc. +/* Various utility functions. + Copyright (C) 2003 Free Software Foundation, Inc. -This file is part of Wget. +This file is part of GNU Wget. -This program is free software; you can redistribute it and/or modify +GNU Wget is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. -This program is distributed in the hope that it will be useful, +GNU Wget is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ +along with Wget; if not, write to the Free Software +Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +In addition, as a special exception, the Free Software Foundation +gives permission to link the code of its release of Wget with the +OpenSSL project's "OpenSSL" library (or with modified versions of it +that use the same license as the "OpenSSL" library), and distribute +the linked executables. You must obey the GNU General Public License +in all respects for all of the code used other than "OpenSSL". If you +modify this file, you may extend this exception to your version of the +file, but you are not obligated to do so. If you do not wish to do +so, delete this exception statement from your version. */ #include @@ -50,259 +60,72 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #include #include -#include "wget.h" -#include "utils.h" -#include "fnmatch.h" -#include "hash.h" - -#ifndef errno -extern int errno; +/* For TIOCGWINSZ and friends: */ +#ifdef HAVE_SYS_IOCTL_H +# include +#endif +#ifdef HAVE_TERMIOS_H +# include #endif -/* This section implements several wrappers around the basic - allocation routines. This is done for two reasons: first, so that - the callers of these functions need not consistently check for - errors. If there is not enough virtual memory for running Wget, - something is seriously wrong, and Wget exits with an appropriate - error message. - - The second reason why these are useful is that, if DEBUG_MALLOC is - defined, they also provide a handy (if crude) malloc debugging - interface that checks memory leaks. */ - -/* Croak the fatal memory error and bail out with non-zero exit - status. */ -static void -memfatal (const char *what) -{ - /* HACK: expose save_log_p from log.c, so we can turn it off in - order to prevent saving the log. Saving the log is dangerous - because logprintf() and logputs() can call malloc(), so this - could infloop. When logging is turned off, infloop can no longer - happen. - - #### This is no longer really necessary because the new routines - in log.c cons only if the line exceeds eighty characters. But - this can come at the end of a line, so it's OK to be careful. - - On a more serious note, it would be good to have a - log_forced_shutdown() routine that exposes this cleanly. */ - extern int save_log_p; - - save_log_p = 0; - logprintf (LOG_ALWAYS, _("%s: %s: Not enough memory.\n"), exec_name, what); - exit (1); -} - -/* These functions end with _real because they need to be - distinguished from the debugging functions, and from the macros. - Explanation follows: - - If memory debugging is not turned on, wget.h defines these: - - #define xmalloc xmalloc_real - #define xrealloc xrealloc_real - #define xstrdup xstrdup_real - #define xfree free - - In case of memory debugging, the definitions are a bit more - complex, because we want to provide more information, *and* we want - to call the debugging code. (The former is the reason why xmalloc - and friends need to be macros in the first place.) Then it looks - like this: - - #define xmalloc(a) xmalloc_debug (a, __FILE__, __LINE__) - #define xfree(a) xfree_debug (a, __FILE__, __LINE__) - #define xrealloc(a, b) xrealloc_debug (a, b, __FILE__, __LINE__) - #define xstrdup(a) xstrdup_debug (a, __FILE__, __LINE__) +/* Needed for run_with_timeout. */ +#undef USE_SIGNAL_TIMEOUT +#ifdef HAVE_SIGNAL_H +# include +#endif +#ifdef HAVE_SETJMP_H +# include +#endif - Each of the *_debug function does its magic and calls the real one. */ +#ifndef HAVE_SIGSETJMP +/* If sigsetjmp is a macro, configure won't pick it up. */ +# ifdef sigsetjmp +# define HAVE_SIGSETJMP +# endif +#endif -#ifdef DEBUG_MALLOC -# define STATIC_IF_DEBUG static -#else -# define STATIC_IF_DEBUG +#ifdef HAVE_SIGNAL +# ifdef HAVE_SIGSETJMP +# define USE_SIGNAL_TIMEOUT +# endif +# ifdef HAVE_SIGBLOCK +# define USE_SIGNAL_TIMEOUT +# endif #endif -STATIC_IF_DEBUG void * -xmalloc_real (size_t size) -{ - void *ptr = malloc (size); - if (!ptr) - memfatal ("malloc"); - return ptr; -} +#include "wget.h" +#include "utils.h" +#include "hash.h" -STATIC_IF_DEBUG void * -xrealloc_real (void *ptr, size_t newsize) -{ - void *newptr; +#ifndef errno +extern int errno; +#endif - /* Not all Un*xes have the feature of realloc() that calling it with - a NULL-pointer is the same as malloc(), but it is easy to - simulate. */ - if (ptr) - newptr = realloc (ptr, newsize); - else - newptr = malloc (newsize); - if (!newptr) - memfatal ("realloc"); - return newptr; -} +/* Utility function: like xstrdup(), but also lowercases S. */ -STATIC_IF_DEBUG char * -xstrdup_real (const char *s) +char * +xstrdup_lower (const char *s) { - char *copy; - -#ifndef HAVE_STRDUP - int l = strlen (s); - copy = malloc (l + 1); - if (!copy) - memfatal ("strdup"); - memcpy (copy, s, l + 1); -#else /* HAVE_STRDUP */ - copy = strdup (s); - if (!copy) - memfatal ("strdup"); -#endif /* HAVE_STRDUP */ - + char *copy = xstrdup (s); + char *p = copy; + for (; *p; p++) + *p = TOLOWER (*p); return copy; } -#ifdef DEBUG_MALLOC - -/* Crude home-grown routines for debugging some malloc-related - problems. Featured: - - * Counting the number of malloc and free invocations, and reporting - the "balance", i.e. how many times more malloc was called than it - was the case with free. - - * Making malloc store its entry into a simple array and free remove - stuff from that array. At the end, print the pointers which have - not been freed, along with the source file and the line number. - This also has the side-effect of detecting freeing memory that - was never allocated. - - Note that this kind of memory leak checking strongly depends on - every malloc() being followed by a free(), even if the program is - about to finish. Wget is careful to free the data structure it - allocated in init.c. */ - -static int malloc_count, free_count; +/* Return a count of how many times CHR occurs in STRING. */ -static struct { - char *ptr; - const char *file; - int line; -} malloc_debug[100000]; - -/* Both register_ptr and unregister_ptr take O(n) operations to run, - which can be a real problem. It would be nice to use a hash table - for malloc_debug, but the functions in hash.c are not suitable - because they can call malloc() themselves. Maybe it would work if - the hash table were preallocated to a huge size, and if we set the - rehash threshold to 1.0. */ - -/* Register PTR in malloc_debug. Abort if this is not possible - (presumably due to the number of current allocations exceeding the - size of malloc_debug.) */ - -static void -register_ptr (void *ptr, const char *file, int line) -{ - int i; - for (i = 0; i < ARRAY_SIZE (malloc_debug); i++) - if (malloc_debug[i].ptr == NULL) - { - malloc_debug[i].ptr = ptr; - malloc_debug[i].file = file; - malloc_debug[i].line = line; - return; - } - abort (); -} - -/* Unregister PTR from malloc_debug. Abort if PTR is not present in - malloc_debug. (This catches calling free() with a bogus pointer.) */ - -static void -unregister_ptr (void *ptr) -{ - int i; - for (i = 0; i < ARRAY_SIZE (malloc_debug); i++) - if (malloc_debug[i].ptr == ptr) - { - malloc_debug[i].ptr = NULL; - return; - } - abort (); -} - -/* Print the malloc debug stats that can be gathered from the above - information. Currently this is the count of mallocs, frees, the - difference between the two, and the dump of the contents of - malloc_debug. The last part are the memory leaks. */ - -void -print_malloc_debug_stats (void) -{ - int i; - printf ("\nMalloc: %d\nFree: %d\nBalance: %d\n\n", - malloc_count, free_count, malloc_count - free_count); - for (i = 0; i < ARRAY_SIZE (malloc_debug); i++) - if (malloc_debug[i].ptr != NULL) - printf ("0x%08ld: %s:%d\n", (long)malloc_debug[i].ptr, - malloc_debug[i].file, malloc_debug[i].line); -} - -void * -xmalloc_debug (size_t size, const char *source_file, int source_line) -{ - void *ptr = xmalloc_real (size); - ++malloc_count; - register_ptr (ptr, source_file, source_line); - return ptr; -} - -void -xfree_debug (void *ptr, const char *source_file, int source_line) -{ - assert (ptr != NULL); - ++free_count; - unregister_ptr (ptr); - free (ptr); -} - -void * -xrealloc_debug (void *ptr, size_t newsize, const char *source_file, int source_line) -{ - void *newptr = xrealloc_real (ptr, newsize); - if (!ptr) - { - ++malloc_count; - register_ptr (newptr, source_file, source_line); - } - else if (newptr != ptr) - { - unregister_ptr (ptr); - register_ptr (newptr, source_file, source_line); - } - return newptr; -} - -char * -xstrdup_debug (const char *s, const char *source_file, int source_line) +int +count_char (const char *string, char chr) { - char *copy = xstrdup_real (s); - ++malloc_count; - register_ptr (copy, source_file, source_line); - return copy; + const char *p; + int count = 0; + for (p = string; *p; p++) + if (*p == chr) + ++count; + return count; } -#endif /* DEBUG_MALLOC */ - /* Copy the string formed by two pointers (one on the beginning, other on the char after the last char) to a new, malloc-ed location. 0-terminate it. */ @@ -403,30 +226,6 @@ datetime_str (time_t *tm) ptm->tm_hour, ptm->tm_min, ptm->tm_sec); return output; } - -/* Returns an error message for ERRNUM. #### This requires more work. - This function, as well as the whole error system, is very - ill-conceived. */ -const char * -uerrmsg (uerr_t errnum) -{ - switch (errnum) - { - case URLUNKNOWN: - return _("Unknown/unsupported protocol"); - break; - case URLBADPORT: - return _("Invalid port specification"); - break; - case URLBADHOST: - return _("Invalid host name"); - break; - default: - abort (); - /* $@#@#$ compiler. */ - return NULL; - } -} /* The Windows versions of the following two functions are defined in mswindows.c. */ @@ -441,7 +240,7 @@ fork_to_background (void) if (!opt.lfilename) { - opt.lfilename = unique_name (DEFAULT_LOGFILE); + opt.lfilename = unique_name (DEFAULT_LOGFILE, 0); changedp = 1; } pid = fork (); @@ -454,137 +253,19 @@ fork_to_background (void) else if (pid != 0) { /* parent, no error */ - printf (_("Continuing in background.\n")); + printf (_("Continuing in background, pid %d.\n"), (int)pid); if (changedp) printf (_("Output will be written to `%s'.\n"), opt.lfilename); - exit (0); - } - /* child: keep running */ -} -#endif /* not WINDOWS */ - -/* Canonicalize PATH, and return a new path. The new path differs from PATH - in that: - Multple `/'s are collapsed to a single `/'. - Leading `./'s and trailing `/.'s are removed. - Trailing `/'s are removed. - Non-leading `../'s and trailing `..'s are handled by removing - portions of the path. - - E.g. "a/b/c/./../d/.." will yield "a/b". This function originates - from GNU Bash. - - Changes for Wget: - Always use '/' as stub_char. - Don't check for local things using canon_stat. - Change the original string instead of strdup-ing. - React correctly when beginning with `./' and `../'. */ -void -path_simplify (char *path) -{ - register int i, start, ddot; - char stub_char; - - if (!*path) - return; - - /*stub_char = (*path == '/') ? '/' : '.';*/ - stub_char = '/'; - - /* Addition: Remove all `./'-s preceding the string. If `../'-s - precede, put `/' in front and remove them too. */ - i = 0; - ddot = 0; - while (1) - { - if (path[i] == '.' && path[i + 1] == '/') - i += 2; - else if (path[i] == '.' && path[i + 1] == '.' && path[i + 2] == '/') - { - i += 3; - ddot = 1; - } - else - break; + exit (0); /* #### should we use _exit()? */ } - if (i) - strcpy (path, path + i - ddot); - /* Replace single `.' or `..' with `/'. */ - if ((path[0] == '.' && path[1] == '\0') - || (path[0] == '.' && path[1] == '.' && path[2] == '\0')) - { - path[0] = stub_char; - path[1] = '\0'; - return; - } - /* Walk along PATH looking for things to compact. */ - i = 0; - while (1) - { - if (!path[i]) - break; - - while (path[i] && path[i] != '/') - i++; - - start = i++; - - /* If we didn't find any slashes, then there is nothing left to do. */ - if (!path[start]) - break; - - /* Handle multiple `/'s in a row. */ - while (path[i] == '/') - i++; - - if ((start + 1) != i) - { - strcpy (path + start + 1, path + i); - i = start + 1; - } - - /* Check for trailing `/'. */ - if (start && !path[i]) - { - zero_last: - path[--i] = '\0'; - break; - } - - /* Check for `../', `./' or trailing `.' by itself. */ - if (path[i] == '.') - { - /* Handle trailing `.' by itself. */ - if (!path[i + 1]) - goto zero_last; - - /* Handle `./'. */ - if (path[i + 1] == '/') - { - strcpy (path + i, path + i + 1); - i = (start < 0) ? 0 : start; - continue; - } - - /* Handle `../' or trailing `..' by itself. */ - if (path[i + 1] == '.' && - (path[i + 2] == '/' || !path[i + 2])) - { - while (--start > -1 && path[start] != '/'); - strcpy (path + start + 1, path + i + 2); - i = (start < 0) ? 0 : start; - continue; - } - } /* path == '.' */ - } /* while */ - - if (!*path) - { - *path = stub_char; - path[1] = '\0'; - } + /* child: give up the privileges and keep running. */ + setsid (); + freopen ("/dev/null", "r", stdin); + freopen ("/dev/null", "w", stdout); + freopen ("/dev/null", "w", stderr); } +#endif /* not WINDOWS */ /* "Touch" FILE, i.e. make its atime and mtime equal to the time specified with TM. */ @@ -653,39 +334,73 @@ file_non_directory_p (const char *path) return S_ISDIR (buf.st_mode) ? 0 : 1; } -/* Return a unique filename, given a prefix and count */ +/* Return the size of file named by FILENAME, or -1 if it cannot be + opened or seeked into. */ +long +file_size (const char *filename) +{ + long size; + /* We use fseek rather than stat to determine the file size because + that way we can also verify whether the file is readable. + Inspired by the POST patch by Arnaud Wylie. */ + FILE *fp = fopen (filename, "rb"); + if (!fp) + return -1; + fseek (fp, 0, SEEK_END); + size = ftell (fp); + fclose (fp); + return size; +} + +/* stat file names named PREFIX.1, PREFIX.2, etc., until one that + doesn't exist is found. Return a freshly allocated copy of the + unused file name. */ + static char * -unique_name_1 (const char *fileprefix, int count) +unique_name_1 (const char *prefix) { - char *filename; + int count = 1; + int plen = strlen (prefix); + char *template = (char *)alloca (plen + 1 + 24); + char *template_tail = template + plen; - if (count) - { - filename = (char *)xmalloc (strlen (fileprefix) + numdigit (count) + 2); - sprintf (filename, "%s.%d", fileprefix, count); - } - else - filename = xstrdup (fileprefix); + memcpy (template, prefix, plen); + *template_tail++ = '.'; - if (!file_exists_p (filename)) - return filename; - else - { - xfree (filename); - return NULL; - } + do + number_to_string (template_tail, count++); + while (file_exists_p (template)); + + return xstrdup (template); } -/* Return a unique file name, based on PREFIX. */ +/* Return a unique file name, based on FILE. + + More precisely, if FILE doesn't exist, it is returned unmodified. + If not, FILE.1 is tried, then FILE.2, etc. The first FILE. + file name that doesn't exist is returned. + + The resulting file is not created, only verified that it didn't + exist at the point in time when the function was called. + Therefore, where security matters, don't rely that the file created + by this function exists until you open it with O_EXCL or + something. + + If ALLOW_PASSTHROUGH is 0, it always returns a freshly allocated + string. Otherwise, it may return FILE if the file doesn't exist + (and therefore doesn't need changing). */ + char * -unique_name (const char *prefix) +unique_name (const char *file, int allow_passthrough) { - char *file = NULL; - int count = 0; - - while (!file) - file = unique_name_1 (prefix, count++); - return file; + /* If the FILE itself doesn't exist, return it without + modification. */ + if (!file_exists_p (file)) + return allow_passthrough ? (char *)file : xstrdup (file); + + /* Otherwise, find a numeric suffix that results in unused file name + and return it. */ + return unique_name_1 (file); } /* Create DIRECTORY. If some of the pathname components of DIRECTORY @@ -699,6 +414,7 @@ make_directory (const char *directory) { int quit = 0; int i; + int ret = 0; char *dir; /* Make a copy of dir, to be able to write to it. Otherwise, the @@ -714,18 +430,45 @@ make_directory (const char *directory) if (!dir[i]) quit = 1; dir[i] = '\0'; - /* Check whether the directory already exists. */ + /* Check whether the directory already exists. Allow creation of + of intermediate directories to fail, as the initial path components + are not necessarily directories! */ if (!file_exists_p (dir)) - { - if (mkdir (dir, 0777) < 0) - return -1; - } + ret = mkdir (dir, 0777); + else + ret = 0; if (quit) break; else dir[i] = '/'; } - return 0; + return ret; +} + +/* Merge BASE with FILE. BASE can be a directory or a file name, FILE + should be a file name. + + file_merge("/foo/bar", "baz") => "/foo/baz" + file_merge("/foo/bar/", "baz") => "/foo/bar/baz" + file_merge("foo", "bar") => "bar" + + In other words, it's a simpler and gentler version of uri_merge_1. */ + +char * +file_merge (const char *base, const char *file) +{ + char *result; + const char *cut = (const char *)strrchr (base, '/'); + + if (!cut) + return xstrdup (file); + + result = (char *)xmalloc (cut - base + 1 + strlen (file) + 1); + memcpy (result, base, cut - base); + result[cut - base] = '/'; + strcpy (result + (cut - base) + 1, file); + + return result; } static int in_acclist PARAMS ((const char *const *, const char *, int)); @@ -810,20 +553,37 @@ accdir (const char *directory, enum accd flags) return 1; } -/* Match the end of STRING against PATTERN. For instance: +/* Return non-zero if STRING ends with TAIL. For instance: - match_backwards ("abc", "bc") -> 1 - match_backwards ("abc", "ab") -> 0 - match_backwards ("abc", "abc") -> 1 */ -static int -match_backwards (const char *string, const char *pattern) + match_tail ("abc", "bc", 0) -> 1 + match_tail ("abc", "ab", 0) -> 0 + match_tail ("abc", "abc", 0) -> 1 + + If FOLD_CASE_P is non-zero, the comparison will be + case-insensitive. */ + +int +match_tail (const char *string, const char *tail, int fold_case_p) { int i, j; - for (i = strlen (string), j = strlen (pattern); i >= 0 && j >= 0; i--, j--) - if (string[i] != pattern[j]) - break; - /* If the pattern was exhausted, the match was succesful. */ + /* We want this to be fast, so we code two loops, one with + case-folding, one without. */ + + if (!fold_case_p) + { + for (i = strlen (string), j = strlen (tail); i >= 0 && j >= 0; i--, j--) + if (string[i] != tail[j]) + break; + } + else + { + for (i = strlen (string), j = strlen (tail); i >= 0 && j >= 0; i--, j--) + if (TOLOWER (string[i]) != TOLOWER (tail[j])) + break; + } + + /* If the tail was exhausted, the match was succesful. */ if (j == -1) return 1; else @@ -831,7 +591,7 @@ match_backwards (const char *string, const char *pattern) } /* Checks whether string S matches each element of ACCEPTS. A list - element are matched either with fnmatch() or match_backwards(), + element are matched either with fnmatch() or match_tail(), according to whether the element contains wildcards or not. If the BACKWARD is 0, don't do backward comparison -- just compare @@ -852,7 +612,7 @@ in_acclist (const char *const *accepts, const char *s, int backward) { if (backward) { - if (match_backwards (s, *accepts)) + if (match_tail (s, *accepts, 0)) return 1; } else @@ -865,7 +625,7 @@ in_acclist (const char *const *accepts, const char *s, int backward) return 0; } -/* Return the malloc-ed suffix of STR. For instance: +/* Return the location of STR's suffix (file extension). Examples: suffix ("foo.bar") -> "bar" suffix ("foo.bar.baz") -> "baz" suffix ("/foo/bar") -> NULL @@ -875,18 +635,59 @@ suffix (const char *str) { int i; - for (i = strlen (str); i && str[i] != '/' && str[i] != '.'; i--); + for (i = strlen (str); i && str[i] != '/' && str[i] != '.'; i--) + ; + if (str[i++] == '.') - return xstrdup (str + i); + return (char *)str + i; else return NULL; } -/* Read a line from FP. The function reallocs the storage as needed - to accomodate for any length of the line. Reallocs are done - storage exponentially, doubling the storage after each overflow to - minimize the number of calls to realloc() and fgets(). The newline - character at the end of line is retained. +/* Return non-zero if S contains globbing wildcards (`*', `?', `[' or + `]'). */ + +int +has_wildcards_p (const char *s) +{ + for (; *s; s++) + if (*s == '*' || *s == '?' || *s == '[' || *s == ']') + return 1; + return 0; +} + +/* Return non-zero if FNAME ends with a typical HTML suffix. The + following (case-insensitive) suffixes are presumed to be HTML files: + + html + htm + ?html (`?' matches one character) + + #### CAVEAT. This is not necessarily a good indication that FNAME + refers to a file that contains HTML! */ +int +has_html_suffix_p (const char *fname) +{ + char *suf; + + if ((suf = suffix (fname)) == NULL) + return 0; + if (!strcasecmp (suf, "html")) + return 1; + if (!strcasecmp (suf, "htm")) + return 1; + if (suf[0] && !strcasecmp (suf + 1, "html")) + return 1; + return 0; +} + +/* Read a line from FP and return the pointer to freshly allocated + storage. The storage space is obtained through malloc() and should + be freed with free() when it is no longer needed. + + The length of the line is not limited, except by available memory. + The newline character at the end of line is retained. The line is + terminated with a zero character. After end-of-file is encountered without anything being read, NULL is returned. NULL is also returned on error. To distinguish @@ -896,15 +697,20 @@ char * read_whole_line (FILE *fp) { int length = 0; - int bufsize = 81; + int bufsize = 82; char *line = (char *)xmalloc (bufsize); while (fgets (line + length, bufsize - length, fp)) { length += strlen (line + length); - assert (length > 0); + if (length == 0) + /* Possible for example when reading from a binary file where + a line begins with \0. */ + continue; + if (line[length - 1] == '\n') break; + /* fgets() guarantees to read the whole line, or to use up the space we've given it. We can double the buffer unconditionally. */ @@ -962,7 +768,7 @@ read_file (const char *file) fd = open (file, O_RDONLY); if (fd < 0) return NULL; - fm = xmalloc (sizeof (struct file_memory)); + fm = xnew (struct file_memory); #ifdef HAVE_MMAP { @@ -1011,7 +817,7 @@ read_file (const char *file) /* Normally, we grow SIZE exponentially to make the number of calls to read() and realloc() logarithmic in relation to file size. However, read() can read an amount of data - smaller than requested, and it would be unreasonably to + smaller than requested, and it would be unreasonable to double SIZE every time *something* was read. Therefore, we double SIZE only when the length exceeds half of the entire allocated size. */ @@ -1130,7 +936,7 @@ merge_vecs (char **v1, char **v2) slist * slist_append (slist *l, const char *s) { - slist *newel = (slist *)xmalloc (sizeof (slist)); + slist *newel = xnew (slist); slist *beg = l; newel->string = xstrdup (s); @@ -1150,7 +956,7 @@ slist_append (slist *l, const char *s) slist * slist_prepend (slist *l, const char *s) { - slist *newel = (slist *)xmalloc (sizeof (slist)); + slist *newel = xnew (slist); newel->string = xstrdup (s); newel->next = l; return newel; @@ -1206,7 +1012,7 @@ string_set_add (struct hash_table *ht, const char *s) /* First check whether the set element already exists. If it does, do nothing so that we don't have to free() the old element and then strdup() a new one. */ - if (hash_table_exists (ht, s)) + if (hash_table_contains (ht, s)) return; /* We use "1" as value. It provides us a useful and clear arbitrary @@ -1216,12 +1022,12 @@ string_set_add (struct hash_table *ht, const char *s) hash_table_put (ht, xstrdup (s), "1"); } -/* Synonym for hash_table_exists... */ +/* Synonym for hash_table_contains... */ int -string_set_exists (struct hash_table *ht, const char *s) +string_set_contains (struct hash_table *ht, const char *s) { - return hash_table_exists (ht, s); + return hash_table_contains (ht, s); } static int @@ -1255,13 +1061,13 @@ free_keys_and_values (struct hash_table *ht) } -/* Engine for legible and legible_very_long; this function works on - strings. */ +/* Engine for legible and legible_large_int; add thousand separators + to numbers printed in strings. */ static char * legible_1 (const char *repr) { - static char outbuf[128]; + static char outbuf[48]; int i, i1, mod; char *outptr; const char *inptr; @@ -1269,7 +1075,9 @@ legible_1 (const char *repr) /* Reset the pointers. */ outptr = outbuf; inptr = repr; - /* If the number is negative, shift the pointers. */ + + /* Ignore the sign for the purpose of adding thousand + separators. */ if (*inptr == '-') { *outptr++ = '-'; @@ -1294,77 +1102,63 @@ legible_1 (const char *repr) } /* Legible -- return a static pointer to the legibly printed long. */ + char * legible (long l) { char inbuf[24]; /* Print the number into the buffer. */ - long_to_string (inbuf, l); + number_to_string (inbuf, l); return legible_1 (inbuf); } -/* Write a string representation of NUMBER into the provided buffer. - We cannot use sprintf() because we cannot be sure whether the - platform supports printing of what we chose for VERY_LONG_TYPE. - - Example: Gcc supports `long long' under many platforms, but on many - of those the native libc knows nothing of it and therefore cannot - print it. - - How long BUFFER needs to be depends on the platform and the content - of NUMBER. For 64-bit VERY_LONG_TYPE (the most common case), 24 - bytes are sufficient. Using more might be a good idea. +/* Write a string representation of LARGE_INT NUMBER into the provided + buffer. The buffer should be able to accept 24 characters, + including the terminating zero. - This function does not go through the hoops that long_to_string - goes to because it doesn't need to be fast. (It's called perhaps - once in a Wget run.) */ + It would be dangerous to use sprintf, because the code wouldn't + work on a machine with gcc-provided long long support, but without + libc support for "%lld". However, such platforms will typically + not have snprintf and will use our version, which does support + "%lld" where long longs are available. */ static void -very_long_to_string (char *buffer, VERY_LONG_TYPE number) +large_int_to_string (char *buffer, LARGE_INT number) { - int i = 0; - int j; - - /* Print the number backwards... */ - do - { - buffer[i++] = '0' + number % 10; - number /= 10; - } - while (number); - - /* ...and reverse the order of the digits. */ - for (j = 0; j < i / 2; j++) - { - char c = buffer[j]; - buffer[j] = buffer[i - 1 - j]; - buffer[i - 1 - j] = c; - } - buffer[i] = '\0'; + snprintf (buffer, 24, LARGE_INT_FMT, number); } -/* The same as legible(), but works on VERY_LONG_TYPE. See sysdep.h. */ +/* The same as legible(), but works on LARGE_INT. */ + char * -legible_very_long (VERY_LONG_TYPE l) +legible_large_int (LARGE_INT l) { - char inbuf[128]; - /* Print the number into the buffer. */ - very_long_to_string (inbuf, l); + char inbuf[48]; + large_int_to_string (inbuf, l); return legible_1 (inbuf); } /* Count the digits in a (long) integer. */ int -numdigit (long a) +numdigit (long number) { - int res = 1; - if (a < 0) - a = -a; - while ((a /= 10) != 0) - ++res; - return res; + int cnt = 1; + if (number < 0) + { + number = -number; + ++cnt; + } + while ((number /= 10) > 0) + ++cnt; + return cnt; } +/* A half-assed implementation of INT_MAX on machines that don't + bother to define one. */ +#ifndef INT_MAX +# define INT_MAX ((int) ~((unsigned)1 << 8 * sizeof (int) - 1)) +#endif + #define ONE_DIGIT(figure) *p++ = n / (figure) + '0' #define ONE_DIGIT_ADVANCE(figure) (ONE_DIGIT (figure), n %= (figure)) @@ -1379,7 +1173,7 @@ numdigit (long a) #define DIGITS_9(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_8 ((figure) / 10) #define DIGITS_10(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_9 ((figure) / 10) -/* DIGITS_<11-20> are only used on 64-bit machines. */ +/* DIGITS_<11-20> are only used on machines with 64-bit longs. */ #define DIGITS_11(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_10 ((figure) / 10) #define DIGITS_12(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_11 ((figure) / 10) @@ -1391,21 +1185,26 @@ numdigit (long a) #define DIGITS_18(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_17 ((figure) / 10) #define DIGITS_19(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_18 ((figure) / 10) -/* Print NUMBER to BUFFER in base 10. This is completely equivalent - to `sprintf(buffer, "%ld", number)', only much faster. +/* Print NUMBER to BUFFER in base 10. This should be completely + equivalent to `sprintf(buffer, "%ld", number)', only much faster. The speedup may make a difference in programs that frequently convert numbers to strings. Some implementations of sprintf, particularly the one in GNU libc, have been known to be extremely slow compared to this function. - BUFFER should accept as many bytes as you expect the number to take - up. On 64-bit machines, the maximum needed size is 24 bytes. That - includes all the digits, as well as the `-' sign for negative - numbers and the trailing \0. */ + Return the pointer to the location where the terminating zero was + printed. (Equivalent to calling buffer+strlen(buffer) after the + function is done.) -void -long_to_string (char *buffer, long number) + BUFFER should be big enough to accept as many bytes as you expect + the number to take up. On machines with 64-bit longs the maximum + needed size is 24 bytes. That includes the digits needed for the + largest 64-bit number, the `-' sign in case it's negative, and the + terminating '\0'. */ + +char * +number_to_string (char *buffer, long number) { char *p = buffer; long n = number; @@ -1414,10 +1213,20 @@ long_to_string (char *buffer, long number) /* We are running in a strange or misconfigured environment. Let sprintf cope with it. */ sprintf (buffer, "%ld", n); + p += strlen (buffer); #else /* (SIZEOF_LONG == 4) || (SIZEOF_LONG == 8) */ if (n < 0) { + if (n < -INT_MAX) + { + /* We cannot print a '-' and assign -n to n because -n would + overflow. Let sprintf deal with this border case. */ + sprintf (buffer, "%ld", n); + p += strlen (buffer); + return p; + } + *p++ = '-'; n = -n; } @@ -1449,6 +1258,8 @@ long_to_string (char *buffer, long number) *p = '\0'; #endif /* (SIZEOF_LONG == 4) || (SIZEOF_LONG == 8) */ + + return p; } #undef ONE_DIGIT @@ -1500,29 +1311,39 @@ long_to_string (char *buffer, long number) # endif #endif /* not WINDOWS */ -struct wget_timer { #ifdef TIMER_GETTIMEOFDAY - long secs; - long usecs; +typedef struct timeval wget_sys_time; #endif #ifdef TIMER_TIME - time_t secs; +typedef time_t wget_sys_time; #endif #ifdef TIMER_WINDOWS - ULARGE_INTEGER wintime; +typedef ULARGE_INTEGER wget_sys_time; #endif + +struct wget_timer { + /* The starting point in time which, subtracted from the current + time, yields elapsed time. */ + wget_sys_time start; + + /* The most recent elapsed time, calculated by wtimer_elapsed(). + Measured in milliseconds. */ + double elapsed_last; + + /* Approximately, the time elapsed between the true start of the + measurement and the time represented by START. */ + double elapsed_pre_start; }; /* Allocate a timer. It is not legal to do anything with a freshly - allocated timer, except call wtimer_reset(). */ + allocated timer, except call wtimer_reset() or wtimer_delete(). */ struct wget_timer * wtimer_allocate (void) { - struct wget_timer *wt = - (struct wget_timer *)xmalloc (sizeof (struct wget_timer)); + struct wget_timer *wt = xnew (struct wget_timer); return wt; } @@ -1545,85 +1366,133 @@ wtimer_delete (struct wget_timer *wt) xfree (wt); } -/* Reset timer WT. This establishes the starting point from which - wtimer_elapsed() will return the number of elapsed - milliseconds. It is allowed to reset a previously used timer. */ +/* Store system time to WST. */ -void -wtimer_reset (struct wget_timer *wt) +static void +wtimer_sys_set (wget_sys_time *wst) { #ifdef TIMER_GETTIMEOFDAY - struct timeval t; - gettimeofday (&t, NULL); - wt->secs = t.tv_sec; - wt->usecs = t.tv_usec; + gettimeofday (wst, NULL); #endif #ifdef TIMER_TIME - wt->secs = time (NULL); + time (wst); #endif #ifdef TIMER_WINDOWS + /* We use GetSystemTime to get the elapsed time. MSDN warns that + system clock adjustments can skew the output of GetSystemTime + when used as a timer and gives preference to GetTickCount and + high-resolution timers. But GetTickCount can overflow, and hires + timers are typically used for profiling, not for regular time + measurement. Since we handle clock skew anyway, we just use + GetSystemTime. */ FILETIME ft; SYSTEMTIME st; GetSystemTime (&st); + + /* As recommended by MSDN, we convert SYSTEMTIME to FILETIME, copy + FILETIME to ULARGE_INTEGER, and use regular 64-bit integer + arithmetic on that. */ SystemTimeToFileTime (&st, &ft); - wt->wintime.HighPart = ft.dwHighDateTime; - wt->wintime.LowPart = ft.dwLowDateTime; + wst->HighPart = ft.dwHighDateTime; + wst->LowPart = ft.dwLowDateTime; #endif } -/* Return the number of milliseconds elapsed since the timer was last - reset. It is allowed to call this function more than once to get - increasingly higher elapsed values. */ +/* Reset timer WT. This establishes the starting point from which + wtimer_elapsed() will return the number of elapsed + milliseconds. It is allowed to reset a previously used timer. */ -long -wtimer_elapsed (struct wget_timer *wt) +void +wtimer_reset (struct wget_timer *wt) +{ + /* Set the start time to the current time. */ + wtimer_sys_set (&wt->start); + wt->elapsed_last = 0; + wt->elapsed_pre_start = 0; +} + +static double +wtimer_sys_diff (wget_sys_time *wst1, wget_sys_time *wst2) { #ifdef TIMER_GETTIMEOFDAY - struct timeval t; - gettimeofday (&t, NULL); - return (t.tv_sec - wt->secs) * 1000 + (t.tv_usec - wt->usecs) / 1000; + return ((double)(wst1->tv_sec - wst2->tv_sec) * 1000 + + (double)(wst1->tv_usec - wst2->tv_usec) / 1000); #endif #ifdef TIMER_TIME - time_t now = time (NULL); - return 1000 * (now - wt->secs); + return 1000 * (*wst1 - *wst2); #endif #ifdef WINDOWS - FILETIME ft; - SYSTEMTIME st; - ULARGE_INTEGER uli; - GetSystemTime (&st); - SystemTimeToFileTime (&st, &ft); - uli.HighPart = ft.dwHighDateTime; - uli.LowPart = ft.dwLowDateTime; - return (long)((uli.QuadPart - wt->wintime.QuadPart) / 10000); + /* VC++ 6 doesn't support direct cast of uint64 to double. To work + around this, we subtract, then convert to signed, then finally to + double. */ + return (double)(signed __int64)(wst1->QuadPart - wst2->QuadPart) / 10000; #endif } -/* Return the assessed granularity of the timer implementation. This - is important for certain code that tries to deal with "zero" time - intervals. */ +/* Return the number of milliseconds elapsed since the timer was last + reset. It is allowed to call this function more than once to get + increasingly higher elapsed values. These timers handle clock + skew. */ -long +double +wtimer_elapsed (struct wget_timer *wt) +{ + wget_sys_time now; + double elapsed; + + wtimer_sys_set (&now); + elapsed = wt->elapsed_pre_start + wtimer_sys_diff (&now, &wt->start); + + /* Ideally we'd just return the difference between NOW and + wt->start. However, the system timer can be set back, and we + could return a value smaller than when we were last called, even + a negative value. Both of these would confuse the callers, which + expect us to return monotonically nondecreasing values. + + Therefore: if ELAPSED is smaller than its previous known value, + we reset wt->start to the current time and effectively start + measuring from this point. But since we don't want the elapsed + value to start from zero, we set elapsed_pre_start to the last + elapsed time and increment all future calculations by that + amount. */ + + if (elapsed < wt->elapsed_last) + { + wt->start = now; + wt->elapsed_pre_start = wt->elapsed_last; + elapsed = wt->elapsed_last; + } + + wt->elapsed_last = elapsed; + return elapsed; +} + +/* Return the assessed granularity of the timer implementation, in + milliseconds. This is used by code that tries to substitute a + better value for timers that have returned zero. */ + +double wtimer_granularity (void) { #ifdef TIMER_GETTIMEOFDAY - /* Granularity of gettimeofday is hugely architecture-dependent. - However, it appears that on modern machines it is better than - 1ms. */ - return 1; + /* Granularity of gettimeofday varies wildly between architectures. + However, it appears that on modern machines it tends to be better + than 1ms. Assume 100 usecs. (Perhaps the configure process + could actually measure this?) */ + return 0.1; #endif #ifdef TIMER_TIME - /* This is clear. */ return 1000; #endif #ifdef TIMER_WINDOWS - /* ? */ + /* According to MSDN, GetSystemTime returns a broken-down time + structure the smallest member of which are milliseconds. */ return 1; #endif } @@ -1702,3 +1571,283 @@ html_quote_string (const char *s) *p = '\0'; return res; } + +/* Determine the width of the terminal we're running on. If that's + not possible, return 0. */ + +int +determine_screen_width (void) +{ + /* If there's a way to get the terminal size using POSIX + tcgetattr(), somebody please tell me. */ +#ifndef TIOCGWINSZ + return 0; +#else /* TIOCGWINSZ */ + int fd; + struct winsize wsz; + + if (opt.lfilename != NULL) + return 0; + + fd = fileno (stderr); + if (ioctl (fd, TIOCGWINSZ, &wsz) < 0) + return 0; /* most likely ENOTTY */ + + return wsz.ws_col; +#endif /* TIOCGWINSZ */ +} + +/* Return a random number between 0 and MAX-1, inclusive. + + If MAX is greater than the value of RAND_MAX+1 on the system, the + returned value will be in the range [0, RAND_MAX]. This may be + fixed in a future release. + + The random number generator is seeded automatically the first time + it is called. + + This uses rand() for portability. It has been suggested that + random() offers better randomness, but this is not required for + Wget, so I chose to go for simplicity and use rand + unconditionally. + + DO NOT use this for cryptographic purposes. It is only meant to be + used in situations where quality of the random numbers returned + doesn't really matter. */ + +int +random_number (int max) +{ + static int seeded; + double bounded; + int rnd; + + if (!seeded) + { + srand (time (NULL)); + seeded = 1; + } + rnd = rand (); + + /* On systems that don't define RAND_MAX, assume it to be 2**15 - 1, + and enforce that assumption by masking other bits. */ +#ifndef RAND_MAX +# define RAND_MAX 32767 + rnd &= RAND_MAX; +#endif + + /* This is equivalent to rand() % max, but uses the high-order bits + for better randomness on architecture where rand() is implemented + using a simple congruential generator. */ + + bounded = (double)max * rnd / (RAND_MAX + 1.0); + return (int)bounded; +} + +/* Return a random uniformly distributed floating point number in the + [0, 1) range. The precision of returned numbers is 9 digits. + + Modify this to use erand48() where available! */ + +double +random_float (void) +{ + /* We can't rely on any specific value of RAND_MAX, but I'm pretty + sure it's greater than 1000. */ + int rnd1 = random_number (1000); + int rnd2 = random_number (1000); + int rnd3 = random_number (1000); + return rnd1 / 1000.0 + rnd2 / 1000000.0 + rnd3 / 1000000000.0; +} + +#if 0 +/* A debugging function for checking whether an MD5 library works. */ + +#include "gen-md5.h" + +char * +debug_test_md5 (char *buf) +{ + unsigned char raw[16]; + static char res[33]; + unsigned char *p1; + char *p2; + int cnt; + ALLOCA_MD5_CONTEXT (ctx); + + gen_md5_init (ctx); + gen_md5_update ((unsigned char *)buf, strlen (buf), ctx); + gen_md5_finish (ctx, raw); + + p1 = raw; + p2 = res; + cnt = 16; + while (cnt--) + { + *p2++ = XNUM_TO_digit (*p1 >> 4); + *p2++ = XNUM_TO_digit (*p1 & 0xf); + ++p1; + } + *p2 = '\0'; + + return res; +} +#endif + +/* Implementation of run_with_timeout, a generic timeout-forcing + routine for systems with Unix-like signal handling. */ + +#ifdef USE_SIGNAL_TIMEOUT +# ifdef HAVE_SIGSETJMP +# define SETJMP(env) sigsetjmp (env, 1) + +static sigjmp_buf run_with_timeout_env; + +static RETSIGTYPE +abort_run_with_timeout (int sig) +{ + assert (sig == SIGALRM); + siglongjmp (run_with_timeout_env, -1); +} +# else /* not HAVE_SIGSETJMP */ +# define SETJMP(env) setjmp (env) + +static jmp_buf run_with_timeout_env; + +static RETSIGTYPE +abort_run_with_timeout (int sig) +{ + assert (sig == SIGALRM); + /* We don't have siglongjmp to preserve the set of blocked signals; + if we longjumped out of the handler at this point, SIGALRM would + remain blocked. We must unblock it manually. */ + int mask = siggetmask (); + mask &= ~sigmask (SIGALRM); + sigsetmask (mask); + + /* Now it's safe to longjump. */ + longjmp (run_with_timeout_env, -1); +} +# endif /* not HAVE_SIGSETJMP */ + +/* Arrange for SIGALRM to be delivered in TIMEOUT seconds. This uses + setitimer where available, alarm otherwise. + + TIMEOUT should be non-zero. If the timeout value is so small that + it would be rounded to zero, it is rounded to the least legal value + instead (1us for setitimer, 1s for alarm). That ensures that + SIGALRM will be delivered in all cases. */ + +static void +alarm_set (double timeout) +{ +#ifdef ITIMER_REAL + /* Use the modern itimer interface. */ + struct itimerval itv; + xzero (itv); + itv.it_value.tv_sec = (long) timeout; + itv.it_value.tv_usec = 1000000L * (timeout - (long)timeout); + if (itv.it_value.tv_sec == 0 && itv.it_value.tv_usec == 0) + /* Ensure that we wait for at least the minimum interval. + Specifying zero would mean "wait forever". */ + itv.it_value.tv_usec = 1; + setitimer (ITIMER_REAL, &itv, NULL); +#else /* not ITIMER_REAL */ + /* Use the old alarm() interface. */ + int secs = (int) timeout; + if (secs == 0) + /* Round TIMEOUTs smaller than 1 to 1, not to zero. This is + because alarm(0) means "never deliver the alarm", i.e. "wait + forever", which is not what someone who specifies a 0.5s + timeout would expect. */ + secs = 1; + alarm (secs); +#endif /* not ITIMER_REAL */ +} + +/* Cancel the alarm set with alarm_set. */ + +static void +alarm_cancel (void) +{ +#ifdef ITIMER_REAL + struct itimerval disable; + xzero (disable); + setitimer (ITIMER_REAL, &disable, NULL); +#else /* not ITIMER_REAL */ + alarm (0); +#endif /* not ITIMER_REAL */ +} + +/* Call FUN(ARG), but don't allow it to run for more than TIMEOUT + seconds. Returns non-zero if the function was interrupted with a + timeout, zero otherwise. + + This works by setting up SIGALRM to be delivered in TIMEOUT seconds + using setitimer() or alarm(). The timeout is enforced by + longjumping out of the SIGALRM handler. This has several + advantages compared to the traditional approach of relying on + signals causing system calls to exit with EINTR: + + * The callback function is *forcibly* interrupted after the + timeout expires, (almost) regardless of what it was doing and + whether it was in a syscall. For example, a calculation that + takes a long time is interrupted as reliably as an IO + operation. + + * It works with both SYSV and BSD signals because it doesn't + depend on the default setting of SA_RESTART. + + * It doesn't special handler setup beyond a simple call to + signal(). (It does use sigsetjmp/siglongjmp, but they're + optional.) + + The only downside is that, if FUN allocates internal resources that + are normally freed prior to exit from the functions, they will be + lost in case of timeout. */ + +int +run_with_timeout (double timeout, void (*fun) (void *), void *arg) +{ + int saved_errno; + + if (timeout == 0) + { + fun (arg); + return 0; + } + + signal (SIGALRM, abort_run_with_timeout); + if (SETJMP (run_with_timeout_env) != 0) + { + /* Longjumped out of FUN with a timeout. */ + signal (SIGALRM, SIG_DFL); + return 1; + } + alarm_set (timeout); + fun (arg); + + /* Preserve errno in case alarm() or signal() modifies it. */ + saved_errno = errno; + alarm_cancel (); + signal (SIGALRM, SIG_DFL); + errno = saved_errno; + + return 0; +} + +#else /* not USE_SIGNAL_TIMEOUT */ + +#ifndef WINDOWS +/* A stub version of run_with_timeout that just calls FUN(ARG). Don't + define it under Windows, because Windows has its own version of + run_with_timeout that uses threads. */ + +int +run_with_timeout (double timeout, void (*fun) (void *), void *arg) +{ + fun (arg); + return 0; +} +#endif /* not WINDOWS */ +#endif /* not USE_SIGNAL_TIMEOUT */