1 /* Various functions of utilitarian nature.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 In addition, as a special exception, the Free Software Foundation
22 gives permission to link the code of its release of Wget with the
23 OpenSSL project's "OpenSSL" library (or with modified versions of it
24 that use the same license as the "OpenSSL" library), and distribute
25 the linked executables. You must obey the GNU General Public License
26 in all respects for all of the code used other than "OpenSSL". If you
27 modify this file, you may extend this exception to your version of the
28 file, but you are not obligated to do so. If you do not wish to do
29 so, delete this exception statement from your version. */
37 #else /* not HAVE_STRING_H */
39 #endif /* not HAVE_STRING_H */
40 #include <sys/types.h>
45 # include <sys/mman.h>
54 #ifdef HAVE_SYS_UTIME_H
55 # include <sys/utime.h>
59 # include <libc.h> /* for access() */
64 /* For TIOCGWINSZ and friends: */
65 #ifdef HAVE_SYS_IOCTL_H
66 # include <sys/ioctl.h>
72 /* Needed for run_with_timeout. */
73 #undef USE_SIGNAL_TIMEOUT
80 /* If sigsetjmp is a macro, configure won't pick it up. */
82 # define HAVE_SIGSETJMP
85 # ifdef HAVE_SIGSETJMP
86 # define USE_SIGNAL_TIMEOUT
89 # define USE_SIGNAL_TIMEOUT
102 /* This section implements several wrappers around the basic
103 allocation routines. This is done for two reasons: first, so that
104 the callers of these functions need not consistently check for
105 errors. If there is not enough virtual memory for running Wget,
106 something is seriously wrong, and Wget exits with an appropriate
109 The second reason why these are useful is that, if DEBUG_MALLOC is
110 defined, they also provide a handy (if crude) malloc debugging
111 interface that checks memory leaks. */
113 /* Croak the fatal memory error and bail out with non-zero exit
116 memfatal (const char *what)
118 /* Make sure we don't try to store part of the log line, and thus
120 log_set_save_context (0);
121 logprintf (LOG_ALWAYS, _("%s: %s: Not enough memory.\n"), exec_name, what);
125 /* These functions end with _real because they need to be
126 distinguished from the debugging functions, and from the macros.
129 If memory debugging is not turned on, wget.h defines these:
131 #define xmalloc xmalloc_real
132 #define xrealloc xrealloc_real
133 #define xstrdup xstrdup_real
136 In case of memory debugging, the definitions are a bit more
137 complex, because we want to provide more information, *and* we want
138 to call the debugging code. (The former is the reason why xmalloc
139 and friends need to be macros in the first place.) Then it looks
142 #define xmalloc(a) xmalloc_debug (a, __FILE__, __LINE__)
143 #define xfree(a) xfree_debug (a, __FILE__, __LINE__)
144 #define xrealloc(a, b) xrealloc_debug (a, b, __FILE__, __LINE__)
145 #define xstrdup(a) xstrdup_debug (a, __FILE__, __LINE__)
147 Each of the *_debug function does its magic and calls the real one. */
150 # define STATIC_IF_DEBUG static
152 # define STATIC_IF_DEBUG
155 STATIC_IF_DEBUG void *
156 xmalloc_real (size_t size)
158 void *ptr = malloc (size);
164 STATIC_IF_DEBUG void *
165 xrealloc_real (void *ptr, size_t newsize)
169 /* Not all Un*xes have the feature of realloc() that calling it with
170 a NULL-pointer is the same as malloc(), but it is easy to
173 newptr = realloc (ptr, newsize);
175 newptr = malloc (newsize);
177 memfatal ("realloc");
181 STATIC_IF_DEBUG char *
182 xstrdup_real (const char *s)
188 copy = malloc (l + 1);
191 memcpy (copy, s, l + 1);
192 #else /* HAVE_STRDUP */
196 #endif /* HAVE_STRDUP */
203 /* Crude home-grown routines for debugging some malloc-related
206 * Counting the number of malloc and free invocations, and reporting
207 the "balance", i.e. how many times more malloc was called than it
208 was the case with free.
210 * Making malloc store its entry into a simple array and free remove
211 stuff from that array. At the end, print the pointers which have
212 not been freed, along with the source file and the line number.
213 This also has the side-effect of detecting freeing memory that
216 Note that this kind of memory leak checking strongly depends on
217 every malloc() being followed by a free(), even if the program is
218 about to finish. Wget is careful to free the data structure it
219 allocated in init.c. */
221 static int malloc_count, free_count;
227 } malloc_debug[100000];
229 /* Both register_ptr and unregister_ptr take O(n) operations to run,
230 which can be a real problem. It would be nice to use a hash table
231 for malloc_debug, but the functions in hash.c are not suitable
232 because they can call malloc() themselves. Maybe it would work if
233 the hash table were preallocated to a huge size, and if we set the
234 rehash threshold to 1.0. */
236 /* Register PTR in malloc_debug. Abort if this is not possible
237 (presumably due to the number of current allocations exceeding the
238 size of malloc_debug.) */
241 register_ptr (void *ptr, const char *file, int line)
244 for (i = 0; i < ARRAY_SIZE (malloc_debug); i++)
245 if (malloc_debug[i].ptr == NULL)
247 malloc_debug[i].ptr = ptr;
248 malloc_debug[i].file = file;
249 malloc_debug[i].line = line;
255 /* Unregister PTR from malloc_debug. Abort if PTR is not present in
256 malloc_debug. (This catches calling free() with a bogus pointer.) */
259 unregister_ptr (void *ptr)
262 for (i = 0; i < ARRAY_SIZE (malloc_debug); i++)
263 if (malloc_debug[i].ptr == ptr)
265 malloc_debug[i].ptr = NULL;
271 /* Print the malloc debug stats that can be gathered from the above
272 information. Currently this is the count of mallocs, frees, the
273 difference between the two, and the dump of the contents of
274 malloc_debug. The last part are the memory leaks. */
277 print_malloc_debug_stats (void)
280 printf ("\nMalloc: %d\nFree: %d\nBalance: %d\n\n",
281 malloc_count, free_count, malloc_count - free_count);
282 for (i = 0; i < ARRAY_SIZE (malloc_debug); i++)
283 if (malloc_debug[i].ptr != NULL)
284 printf ("0x%08ld: %s:%d\n", (long)malloc_debug[i].ptr,
285 malloc_debug[i].file, malloc_debug[i].line);
289 xmalloc_debug (size_t size, const char *source_file, int source_line)
291 void *ptr = xmalloc_real (size);
293 register_ptr (ptr, source_file, source_line);
298 xfree_debug (void *ptr, const char *source_file, int source_line)
300 assert (ptr != NULL);
302 unregister_ptr (ptr);
307 xrealloc_debug (void *ptr, size_t newsize, const char *source_file, int source_line)
309 void *newptr = xrealloc_real (ptr, newsize);
313 register_ptr (newptr, source_file, source_line);
315 else if (newptr != ptr)
317 unregister_ptr (ptr);
318 register_ptr (newptr, source_file, source_line);
324 xstrdup_debug (const char *s, const char *source_file, int source_line)
326 char *copy = xstrdup_real (s);
328 register_ptr (copy, source_file, source_line);
332 #endif /* DEBUG_MALLOC */
334 /* Utility function: like xstrdup(), but also lowercases S. */
337 xstrdup_lower (const char *s)
339 char *copy = xstrdup (s);
346 /* Return a count of how many times CHR occurs in STRING. */
349 count_char (const char *string, char chr)
353 for (p = string; *p; p++)
359 /* Copy the string formed by two pointers (one on the beginning, other
360 on the char after the last char) to a new, malloc-ed location.
363 strdupdelim (const char *beg, const char *end)
365 char *res = (char *)xmalloc (end - beg + 1);
366 memcpy (res, beg, end - beg);
367 res[end - beg] = '\0';
371 /* Parse a string containing comma-separated elements, and return a
372 vector of char pointers with the elements. Spaces following the
373 commas are ignored. */
375 sepstring (const char *s)
389 res = (char **)xrealloc (res, (i + 2) * sizeof (char *));
390 res[i] = strdupdelim (p, s);
393 /* Skip the blanks following the ','. */
401 res = (char **)xrealloc (res, (i + 2) * sizeof (char *));
402 res[i] = strdupdelim (p, s);
407 /* Return pointer to a static char[] buffer in which zero-terminated
408 string-representation of TM (in form hh:mm:ss) is printed.
410 If TM is non-NULL, the current time-in-seconds will be stored
413 (#### This is misleading: one would expect TM would be used instead
414 of the current time in that case. This design was probably
415 influenced by the design time(2), and should be changed at some
416 points. No callers use non-NULL TM anyway.) */
419 time_str (time_t *tm)
421 static char output[15];
423 time_t secs = time (tm);
427 /* In case of error, return the empty string. Maybe we should
428 just abort if this happens? */
432 ptm = localtime (&secs);
433 sprintf (output, "%02d:%02d:%02d", ptm->tm_hour, ptm->tm_min, ptm->tm_sec);
437 /* Like the above, but include the date: YYYY-MM-DD hh:mm:ss. */
440 datetime_str (time_t *tm)
442 static char output[20]; /* "YYYY-MM-DD hh:mm:ss" + \0 */
444 time_t secs = time (tm);
448 /* In case of error, return the empty string. Maybe we should
449 just abort if this happens? */
453 ptm = localtime (&secs);
454 sprintf (output, "%04d-%02d-%02d %02d:%02d:%02d",
455 ptm->tm_year + 1900, ptm->tm_mon + 1, ptm->tm_mday,
456 ptm->tm_hour, ptm->tm_min, ptm->tm_sec);
460 /* The Windows versions of the following two functions are defined in
465 fork_to_background (void)
468 /* Whether we arrange our own version of opt.lfilename here. */
473 opt.lfilename = unique_name (DEFAULT_LOGFILE);
485 /* parent, no error */
486 printf (_("Continuing in background, pid %d.\n"), (int)pid);
488 printf (_("Output will be written to `%s'.\n"), opt.lfilename);
489 exit (0); /* #### should we use _exit()? */
492 /* child: give up the privileges and keep running. */
494 freopen ("/dev/null", "r", stdin);
495 freopen ("/dev/null", "w", stdout);
496 freopen ("/dev/null", "w", stderr);
498 #endif /* not WINDOWS */
500 /* "Touch" FILE, i.e. make its atime and mtime equal to the time
501 specified with TM. */
503 touch (const char *file, time_t tm)
505 #ifdef HAVE_STRUCT_UTIMBUF
506 struct utimbuf times;
507 times.actime = times.modtime = tm;
510 times[0] = times[1] = tm;
513 if (utime (file, ×) == -1)
514 logprintf (LOG_NOTQUIET, "utime(%s): %s\n", file, strerror (errno));
517 /* Checks if FILE is a symbolic link, and removes it if it is. Does
518 nothing under MS-Windows. */
520 remove_link (const char *file)
525 if (lstat (file, &st) == 0 && S_ISLNK (st.st_mode))
527 DEBUGP (("Unlinking %s (symlink).\n", file));
530 logprintf (LOG_VERBOSE, _("Failed to unlink symlink `%s': %s\n"),
531 file, strerror (errno));
536 /* Does FILENAME exist? This is quite a lousy implementation, since
537 it supplies no error codes -- only a yes-or-no answer. Thus it
538 will return that a file does not exist if, e.g., the directory is
539 unreadable. I don't mind it too much currently, though. The
540 proper way should, of course, be to have a third, error state,
541 other than true/false, but that would introduce uncalled-for
542 additional complexity to the callers. */
544 file_exists_p (const char *filename)
547 return access (filename, F_OK) >= 0;
550 return stat (filename, &buf) >= 0;
554 /* Returns 0 if PATH is a directory, 1 otherwise (any kind of file).
555 Returns 0 on error. */
557 file_non_directory_p (const char *path)
560 /* Use lstat() rather than stat() so that symbolic links pointing to
561 directories can be identified correctly. */
562 if (lstat (path, &buf) != 0)
564 return S_ISDIR (buf.st_mode) ? 0 : 1;
567 /* Return the size of file named by FILENAME, or -1 if it cannot be
568 opened or seeked into. */
570 file_size (const char *filename)
573 /* We use fseek rather than stat to determine the file size because
574 that way we can also verify whether the file is readable.
575 Inspired by the POST patch by Arnaud Wylie. */
576 FILE *fp = fopen (filename, "rb");
577 fseek (fp, 0, SEEK_END);
583 /* Return a unique filename, given a prefix and count */
585 unique_name_1 (const char *fileprefix, int count)
591 filename = (char *)xmalloc (strlen (fileprefix) + numdigit (count) + 2);
592 sprintf (filename, "%s.%d", fileprefix, count);
595 filename = xstrdup (fileprefix);
597 if (!file_exists_p (filename))
606 /* Return a unique file name, based on PREFIX. */
608 unique_name (const char *prefix)
614 file = unique_name_1 (prefix, count++);
618 /* Create DIRECTORY. If some of the pathname components of DIRECTORY
619 are missing, create them first. In case any mkdir() call fails,
620 return its error status. Returns 0 on successful completion.
622 The behaviour of this function should be identical to the behaviour
623 of `mkdir -p' on systems where mkdir supports the `-p' option. */
625 make_directory (const char *directory)
632 /* Make a copy of dir, to be able to write to it. Otherwise, the
633 function is unsafe if called with a read-only char *argument. */
634 STRDUP_ALLOCA (dir, directory);
636 /* If the first character of dir is '/', skip it (and thus enable
637 creation of absolute-pathname directories. */
638 for (i = (*dir == '/'); 1; ++i)
640 for (; dir[i] && dir[i] != '/'; i++)
645 /* Check whether the directory already exists. Allow creation of
646 of intermediate directories to fail, as the initial path components
647 are not necessarily directories! */
648 if (!file_exists_p (dir))
649 ret = mkdir (dir, 0777);
660 /* Merge BASE with FILE. BASE can be a directory or a file name, FILE
661 should be a file name.
663 file_merge("/foo/bar", "baz") => "/foo/baz"
664 file_merge("/foo/bar/", "baz") => "/foo/bar/baz"
665 file_merge("foo", "bar") => "bar"
667 In other words, it's a simpler and gentler version of uri_merge_1. */
670 file_merge (const char *base, const char *file)
673 const char *cut = (const char *)strrchr (base, '/');
676 return xstrdup (file);
678 result = (char *)xmalloc (cut - base + 1 + strlen (file) + 1);
679 memcpy (result, base, cut - base);
680 result[cut - base] = '/';
681 strcpy (result + (cut - base) + 1, file);
686 static int in_acclist PARAMS ((const char *const *, const char *, int));
688 /* Determine whether a file is acceptable to be followed, according to
689 lists of patterns to accept/reject. */
691 acceptable (const char *s)
695 while (l && s[l] != '/')
702 return (in_acclist ((const char *const *)opt.accepts, s, 1)
703 && !in_acclist ((const char *const *)opt.rejects, s, 1));
705 return in_acclist ((const char *const *)opt.accepts, s, 1);
707 else if (opt.rejects)
708 return !in_acclist ((const char *const *)opt.rejects, s, 1);
712 /* Compare S1 and S2 frontally; S2 must begin with S1. E.g. if S1 is
713 `/something', frontcmp() will return 1 only if S2 begins with
714 `/something'. Otherwise, 0 is returned. */
716 frontcmp (const char *s1, const char *s2)
718 for (; *s1 && *s2 && (*s1 == *s2); ++s1, ++s2);
722 /* Iterate through STRLIST, and return the first element that matches
723 S, through wildcards or front comparison (as appropriate). */
725 proclist (char **strlist, const char *s, enum accd flags)
729 for (x = strlist; *x; x++)
730 if (has_wildcards_p (*x))
732 if (fnmatch (*x, s, FNM_PATHNAME) == 0)
737 char *p = *x + ((flags & ALLABS) && (**x == '/')); /* Remove '/' */
744 /* Returns whether DIRECTORY is acceptable for download, wrt the
745 include/exclude lists.
747 If FLAGS is ALLABS, the leading `/' is ignored in paths; relative
748 and absolute paths may be freely intermixed. */
750 accdir (const char *directory, enum accd flags)
752 /* Remove starting '/'. */
753 if (flags & ALLABS && *directory == '/')
757 if (!proclist (opt.includes, directory, flags))
762 if (proclist (opt.excludes, directory, flags))
768 /* Return non-zero if STRING ends with TAIL. For instance:
770 match_tail ("abc", "bc", 0) -> 1
771 match_tail ("abc", "ab", 0) -> 0
772 match_tail ("abc", "abc", 0) -> 1
774 If FOLD_CASE_P is non-zero, the comparison will be
778 match_tail (const char *string, const char *tail, int fold_case_p)
782 /* We want this to be fast, so we code two loops, one with
783 case-folding, one without. */
787 for (i = strlen (string), j = strlen (tail); i >= 0 && j >= 0; i--, j--)
788 if (string[i] != tail[j])
793 for (i = strlen (string), j = strlen (tail); i >= 0 && j >= 0; i--, j--)
794 if (TOLOWER (string[i]) != TOLOWER (tail[j]))
798 /* If the tail was exhausted, the match was succesful. */
805 /* Checks whether string S matches each element of ACCEPTS. A list
806 element are matched either with fnmatch() or match_tail(),
807 according to whether the element contains wildcards or not.
809 If the BACKWARD is 0, don't do backward comparison -- just compare
812 in_acclist (const char *const *accepts, const char *s, int backward)
814 for (; *accepts; accepts++)
816 if (has_wildcards_p (*accepts))
818 /* fnmatch returns 0 if the pattern *does* match the
820 if (fnmatch (*accepts, s, 0) == 0)
827 if (match_tail (s, *accepts, 0))
832 if (!strcmp (s, *accepts))
840 /* Return the location of STR's suffix (file extension). Examples:
841 suffix ("foo.bar") -> "bar"
842 suffix ("foo.bar.baz") -> "baz"
843 suffix ("/foo/bar") -> NULL
844 suffix ("/foo.bar/baz") -> NULL */
846 suffix (const char *str)
850 for (i = strlen (str); i && str[i] != '/' && str[i] != '.'; i--)
854 return (char *)str + i;
859 /* Return non-zero if FNAME ends with a typical HTML suffix. The
860 following (case-insensitive) suffixes are presumed to be HTML files:
864 ?html (`?' matches one character)
866 #### CAVEAT. This is not necessarily a good indication that FNAME
867 refers to a file that contains HTML! */
869 has_html_suffix_p (const char *fname)
873 if ((suf = suffix (fname)) == NULL)
875 if (!strcasecmp (suf, "html"))
877 if (!strcasecmp (suf, "htm"))
879 if (suf[0] && !strcasecmp (suf + 1, "html"))
884 /* Read a line from FP and return the pointer to freshly allocated
885 storage. The stoarage space is obtained through malloc() and
886 should be freed with free() when it is no longer needed.
888 The length of the line is not limited, except by available memory.
889 The newline character at the end of line is retained. The line is
890 terminated with a zero character.
892 After end-of-file is encountered without anything being read, NULL
893 is returned. NULL is also returned on error. To distinguish
894 between these two cases, use the stdio function ferror(). */
897 read_whole_line (FILE *fp)
901 char *line = (char *)xmalloc (bufsize);
903 while (fgets (line + length, bufsize - length, fp))
905 length += strlen (line + length);
907 /* Possible for example when reading from a binary file where
908 a line begins with \0. */
911 if (line[length - 1] == '\n')
914 /* fgets() guarantees to read the whole line, or to use up the
915 space we've given it. We can double the buffer
918 line = xrealloc (line, bufsize);
920 if (length == 0 || ferror (fp))
925 if (length + 1 < bufsize)
926 /* Relieve the memory from our exponential greediness. We say
927 `length + 1' because the terminating \0 is not included in
928 LENGTH. We don't need to zero-terminate the string ourselves,
929 though, because fgets() does that. */
930 line = xrealloc (line, length + 1);
934 /* Read FILE into memory. A pointer to `struct file_memory' are
935 returned; use struct element `content' to access file contents, and
936 the element `length' to know the file length. `content' is *not*
937 zero-terminated, and you should *not* read or write beyond the [0,
938 length) range of characters.
940 After you are done with the file contents, call read_file_free to
943 Depending on the operating system and the type of file that is
944 being read, read_file() either mmap's the file into memory, or
945 reads the file into the core using read().
947 If file is named "-", fileno(stdin) is used for reading instead.
948 If you want to read from a real file named "-", use "./-" instead. */
951 read_file (const char *file)
954 struct file_memory *fm;
956 int inhibit_close = 0;
958 /* Some magic in the finest tradition of Perl and its kin: if FILE
959 is "-", just use stdin. */
964 /* Note that we don't inhibit mmap() in this case. If stdin is
965 redirected from a regular file, mmap() will still work. */
968 fd = open (file, O_RDONLY);
971 fm = xmalloc (sizeof (struct file_memory));
976 if (fstat (fd, &buf) < 0)
978 fm->length = buf.st_size;
979 /* NOTE: As far as I know, the callers of this function never
980 modify the file text. Relying on this would enable us to
981 specify PROT_READ and MAP_SHARED for a marginal gain in
982 efficiency, but at some cost to generality. */
983 fm->content = mmap (NULL, fm->length, PROT_READ | PROT_WRITE,
985 if (fm->content == (char *)MAP_FAILED)
995 /* The most common reason why mmap() fails is that FD does not point
996 to a plain file. However, it's also possible that mmap() doesn't
997 work for a particular type of file. Therefore, whenever mmap()
998 fails, we just fall back to the regular method. */
999 #endif /* HAVE_MMAP */
1002 size = 512; /* number of bytes fm->contents can
1003 hold at any given time. */
1004 fm->content = xmalloc (size);
1008 if (fm->length > size / 2)
1010 /* #### I'm not sure whether the whole exponential-growth
1011 thing makes sense with kernel read. On Linux at least,
1012 read() refuses to read more than 4K from a file at a
1013 single chunk anyway. But other Unixes might optimize it
1014 better, and it doesn't *hurt* anything, so I'm leaving
1017 /* Normally, we grow SIZE exponentially to make the number
1018 of calls to read() and realloc() logarithmic in relation
1019 to file size. However, read() can read an amount of data
1020 smaller than requested, and it would be unreasonably to
1021 double SIZE every time *something* was read. Therefore,
1022 we double SIZE only when the length exceeds half of the
1023 entire allocated size. */
1025 fm->content = xrealloc (fm->content, size);
1027 nread = read (fd, fm->content + fm->length, size - fm->length);
1029 /* Successful read. */
1030 fm->length += nread;
1040 if (size > fm->length && fm->length != 0)
1041 /* Due to exponential growth of fm->content, the allocated region
1042 might be much larger than what is actually needed. */
1043 fm->content = xrealloc (fm->content, fm->length);
1050 xfree (fm->content);
1055 /* Release the resources held by FM. Specifically, this calls
1056 munmap() or xfree() on fm->content, depending whether mmap or
1057 malloc/read were used to read in the file. It also frees the
1058 memory needed to hold the FM structure itself. */
1061 read_file_free (struct file_memory *fm)
1066 munmap (fm->content, fm->length);
1071 xfree (fm->content);
1076 /* Free the pointers in a NULL-terminated vector of pointers, then
1077 free the pointer itself. */
1079 free_vec (char **vec)
1090 /* Append vector V2 to vector V1. The function frees V2 and
1091 reallocates V1 (thus you may not use the contents of neither
1092 pointer after the call). If V1 is NULL, V2 is returned. */
1094 merge_vecs (char **v1, char **v2)
1104 /* To avoid j == 0 */
1109 for (i = 0; v1[i]; i++);
1111 for (j = 0; v2[j]; j++);
1112 /* Reallocate v1. */
1113 v1 = (char **)xrealloc (v1, (i + j + 1) * sizeof (char **));
1114 memcpy (v1 + i, v2, (j + 1) * sizeof (char *));
1119 /* A set of simple-minded routines to store strings in a linked list.
1120 This used to also be used for searching, but now we have hash
1123 /* It's a shame that these simple things like linked lists and hash
1124 tables (see hash.c) need to be implemented over and over again. It
1125 would be nice to be able to use the routines from glib -- see
1126 www.gtk.org for details. However, that would make Wget depend on
1127 glib, and I want to avoid dependencies to external libraries for
1128 reasons of convenience and portability (I suspect Wget is more
1129 portable than anything ever written for Gnome). */
1131 /* Append an element to the list. If the list has a huge number of
1132 elements, this can get slow because it has to find the list's
1133 ending. If you think you have to call slist_append in a loop,
1134 think about calling slist_prepend() followed by slist_nreverse(). */
1137 slist_append (slist *l, const char *s)
1139 slist *newel = (slist *)xmalloc (sizeof (slist));
1142 newel->string = xstrdup (s);
1147 /* Find the last element. */
1154 /* Prepend S to the list. Unlike slist_append(), this is O(1). */
1157 slist_prepend (slist *l, const char *s)
1159 slist *newel = (slist *)xmalloc (sizeof (slist));
1160 newel->string = xstrdup (s);
1165 /* Destructively reverse L. */
1168 slist_nreverse (slist *l)
1173 slist *next = l->next;
1181 /* Is there a specific entry in the list? */
1183 slist_contains (slist *l, const char *s)
1185 for (; l; l = l->next)
1186 if (!strcmp (l->string, s))
1191 /* Free the whole slist. */
1193 slist_free (slist *l)
1204 /* Sometimes it's useful to create "sets" of strings, i.e. special
1205 hash tables where you want to store strings as keys and merely
1206 query for their existence. Here is a set of utility routines that
1207 makes that transparent. */
1210 string_set_add (struct hash_table *ht, const char *s)
1212 /* First check whether the set element already exists. If it does,
1213 do nothing so that we don't have to free() the old element and
1214 then strdup() a new one. */
1215 if (hash_table_contains (ht, s))
1218 /* We use "1" as value. It provides us a useful and clear arbitrary
1219 value, and it consumes no memory -- the pointers to the same
1220 string "1" will be shared by all the key-value pairs in all `set'
1222 hash_table_put (ht, xstrdup (s), "1");
1225 /* Synonym for hash_table_contains... */
1228 string_set_contains (struct hash_table *ht, const char *s)
1230 return hash_table_contains (ht, s);
1234 string_set_free_mapper (void *key, void *value_ignored, void *arg_ignored)
1241 string_set_free (struct hash_table *ht)
1243 hash_table_map (ht, string_set_free_mapper, NULL);
1244 hash_table_destroy (ht);
1248 free_keys_and_values_mapper (void *key, void *value, void *arg_ignored)
1255 /* Another utility function: call free() on all keys and values of HT. */
1258 free_keys_and_values (struct hash_table *ht)
1260 hash_table_map (ht, free_keys_and_values_mapper, NULL);
1264 /* Engine for legible and legible_very_long; this function works on
1268 legible_1 (const char *repr)
1270 static char outbuf[128];
1275 /* Reset the pointers. */
1278 /* If the number is negative, shift the pointers. */
1284 /* How many digits before the first separator? */
1285 mod = strlen (inptr) % 3;
1287 for (i = 0; i < mod; i++)
1288 *outptr++ = inptr[i];
1289 /* Now insert the rest of them, putting separator before every
1291 for (i1 = i, i = 0; inptr[i1]; i++, i1++)
1293 if (i % 3 == 0 && i1 != 0)
1295 *outptr++ = inptr[i1];
1297 /* Zero-terminate the string. */
1302 /* Legible -- return a static pointer to the legibly printed long. */
1307 /* Print the number into the buffer. */
1308 number_to_string (inbuf, l);
1309 return legible_1 (inbuf);
1312 /* Write a string representation of NUMBER into the provided buffer.
1313 We cannot use sprintf() because we cannot be sure whether the
1314 platform supports printing of what we chose for VERY_LONG_TYPE.
1316 Example: Gcc supports `long long' under many platforms, but on many
1317 of those the native libc knows nothing of it and therefore cannot
1320 How long BUFFER needs to be depends on the platform and the content
1321 of NUMBER. For 64-bit VERY_LONG_TYPE (the most common case), 24
1322 bytes are sufficient. Using more might be a good idea.
1324 This function does not go through the hoops that long_to_string
1325 goes to because it doesn't aspire to be fast. (It's called perhaps
1326 once in a Wget run.) */
1329 very_long_to_string (char *buffer, VERY_LONG_TYPE number)
1334 /* Print the number backwards... */
1337 buffer[i++] = '0' + number % 10;
1342 /* ...and reverse the order of the digits. */
1343 for (j = 0; j < i / 2; j++)
1346 buffer[j] = buffer[i - 1 - j];
1347 buffer[i - 1 - j] = c;
1352 /* The same as legible(), but works on VERY_LONG_TYPE. See sysdep.h. */
1354 legible_very_long (VERY_LONG_TYPE l)
1357 /* Print the number into the buffer. */
1358 very_long_to_string (inbuf, l);
1359 return legible_1 (inbuf);
1362 /* Count the digits in a (long) integer. */
1364 numdigit (long number)
1372 while ((number /= 10) > 0)
1377 /* A half-assed implementation of INT_MAX on machines that don't
1378 bother to define one. */
1380 # define INT_MAX ((int) ~((unsigned)1 << 8 * sizeof (int) - 1))
1383 #define ONE_DIGIT(figure) *p++ = n / (figure) + '0'
1384 #define ONE_DIGIT_ADVANCE(figure) (ONE_DIGIT (figure), n %= (figure))
1386 #define DIGITS_1(figure) ONE_DIGIT (figure)
1387 #define DIGITS_2(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_1 ((figure) / 10)
1388 #define DIGITS_3(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_2 ((figure) / 10)
1389 #define DIGITS_4(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_3 ((figure) / 10)
1390 #define DIGITS_5(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_4 ((figure) / 10)
1391 #define DIGITS_6(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_5 ((figure) / 10)
1392 #define DIGITS_7(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_6 ((figure) / 10)
1393 #define DIGITS_8(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_7 ((figure) / 10)
1394 #define DIGITS_9(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_8 ((figure) / 10)
1395 #define DIGITS_10(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_9 ((figure) / 10)
1397 /* DIGITS_<11-20> are only used on machines with 64-bit longs. */
1399 #define DIGITS_11(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_10 ((figure) / 10)
1400 #define DIGITS_12(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_11 ((figure) / 10)
1401 #define DIGITS_13(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_12 ((figure) / 10)
1402 #define DIGITS_14(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_13 ((figure) / 10)
1403 #define DIGITS_15(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_14 ((figure) / 10)
1404 #define DIGITS_16(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_15 ((figure) / 10)
1405 #define DIGITS_17(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_16 ((figure) / 10)
1406 #define DIGITS_18(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_17 ((figure) / 10)
1407 #define DIGITS_19(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_18 ((figure) / 10)
1409 /* Print NUMBER to BUFFER in base 10. This should be completely
1410 equivalent to `sprintf(buffer, "%ld", number)', only much faster.
1412 The speedup may make a difference in programs that frequently
1413 convert numbers to strings. Some implementations of sprintf,
1414 particularly the one in GNU libc, have been known to be extremely
1415 slow compared to this function.
1417 Return the pointer to the location where the terminating zero was
1418 printed. (Equivalent to calling buffer+strlen(buffer) after the
1421 BUFFER should be big enough to accept as many bytes as you expect
1422 the number to take up. On machines with 64-bit longs the maximum
1423 needed size is 24 bytes. That includes the digits needed for the
1424 largest 64-bit number, the `-' sign in case it's negative, and the
1425 terminating '\0'. */
1428 number_to_string (char *buffer, long number)
1433 #if (SIZEOF_LONG != 4) && (SIZEOF_LONG != 8)
1434 /* We are running in a strange or misconfigured environment. Let
1435 sprintf cope with it. */
1436 sprintf (buffer, "%ld", n);
1437 p += strlen (buffer);
1438 #else /* (SIZEOF_LONG == 4) || (SIZEOF_LONG == 8) */
1444 /* We cannot print a '-' and assign -n to n because -n would
1445 overflow. Let sprintf deal with this border case. */
1446 sprintf (buffer, "%ld", n);
1447 p += strlen (buffer);
1455 if (n < 10) { DIGITS_1 (1); }
1456 else if (n < 100) { DIGITS_2 (10); }
1457 else if (n < 1000) { DIGITS_3 (100); }
1458 else if (n < 10000) { DIGITS_4 (1000); }
1459 else if (n < 100000) { DIGITS_5 (10000); }
1460 else if (n < 1000000) { DIGITS_6 (100000); }
1461 else if (n < 10000000) { DIGITS_7 (1000000); }
1462 else if (n < 100000000) { DIGITS_8 (10000000); }
1463 else if (n < 1000000000) { DIGITS_9 (100000000); }
1464 #if SIZEOF_LONG == 4
1465 /* ``if (1)'' serves only to preserve editor indentation. */
1466 else if (1) { DIGITS_10 (1000000000); }
1467 #else /* SIZEOF_LONG != 4 */
1468 else if (n < 10000000000L) { DIGITS_10 (1000000000L); }
1469 else if (n < 100000000000L) { DIGITS_11 (10000000000L); }
1470 else if (n < 1000000000000L) { DIGITS_12 (100000000000L); }
1471 else if (n < 10000000000000L) { DIGITS_13 (1000000000000L); }
1472 else if (n < 100000000000000L) { DIGITS_14 (10000000000000L); }
1473 else if (n < 1000000000000000L) { DIGITS_15 (100000000000000L); }
1474 else if (n < 10000000000000000L) { DIGITS_16 (1000000000000000L); }
1475 else if (n < 100000000000000000L) { DIGITS_17 (10000000000000000L); }
1476 else if (n < 1000000000000000000L) { DIGITS_18 (100000000000000000L); }
1477 else { DIGITS_19 (1000000000000000000L); }
1478 #endif /* SIZEOF_LONG != 4 */
1481 #endif /* (SIZEOF_LONG == 4) || (SIZEOF_LONG == 8) */
1487 #undef ONE_DIGIT_ADVANCE
1509 /* Support for timers. */
1511 #undef TIMER_WINDOWS
1512 #undef TIMER_GETTIMEOFDAY
1515 /* Depending on the OS and availability of gettimeofday(), one and
1516 only one of the above constants will be defined. Virtually all
1517 modern Unix systems will define TIMER_GETTIMEOFDAY; Windows will
1518 use TIMER_WINDOWS. TIMER_TIME is a catch-all method for
1519 non-Windows systems without gettimeofday.
1521 #### Perhaps we should also support ftime(), which exists on old
1522 BSD 4.2-influenced systems? (It also existed under MS DOS Borland
1523 C, if memory serves me.) */
1526 # define TIMER_WINDOWS
1527 #else /* not WINDOWS */
1528 # ifdef HAVE_GETTIMEOFDAY
1529 # define TIMER_GETTIMEOFDAY
1533 #endif /* not WINDOWS */
1535 #ifdef TIMER_GETTIMEOFDAY
1536 typedef struct timeval wget_sys_time;
1540 typedef time_t wget_sys_time;
1543 #ifdef TIMER_WINDOWS
1544 typedef ULARGE_INTEGER wget_sys_time;
1548 /* The starting point in time which, subtracted from the current
1549 time, yields elapsed time. */
1550 wget_sys_time start;
1552 /* The most recent elapsed time, calculated by wtimer_elapsed().
1553 Measured in milliseconds. */
1554 double elapsed_last;
1556 /* Approximately, the time elapsed between the true start of the
1557 measurement and the time represented by START. */
1558 double elapsed_pre_start;
1561 /* Allocate a timer. It is not legal to do anything with a freshly
1562 allocated timer, except call wtimer_reset() or wtimer_delete(). */
1565 wtimer_allocate (void)
1567 struct wget_timer *wt =
1568 (struct wget_timer *)xmalloc (sizeof (struct wget_timer));
1572 /* Allocate a new timer and reset it. Return the new timer. */
1577 struct wget_timer *wt = wtimer_allocate ();
1582 /* Free the resources associated with the timer. Its further use is
1586 wtimer_delete (struct wget_timer *wt)
1591 /* Store system time to WST. */
1594 wtimer_sys_set (wget_sys_time *wst)
1596 #ifdef TIMER_GETTIMEOFDAY
1597 gettimeofday (wst, NULL);
1604 #ifdef TIMER_WINDOWS
1607 GetSystemTime (&st);
1608 SystemTimeToFileTime (&st, &ft);
1609 wst->HighPart = ft.dwHighDateTime;
1610 wst->LowPart = ft.dwLowDateTime;
1614 /* Reset timer WT. This establishes the starting point from which
1615 wtimer_elapsed() will return the number of elapsed
1616 milliseconds. It is allowed to reset a previously used timer. */
1619 wtimer_reset (struct wget_timer *wt)
1621 /* Set the start time to the current time. */
1622 wtimer_sys_set (&wt->start);
1623 wt->elapsed_last = 0;
1624 wt->elapsed_pre_start = 0;
1628 wtimer_sys_diff (wget_sys_time *wst1, wget_sys_time *wst2)
1630 #ifdef TIMER_GETTIMEOFDAY
1631 return ((double)(wst1->tv_sec - wst2->tv_sec) * 1000
1632 + (double)(wst1->tv_usec - wst2->tv_usec) / 1000);
1636 return 1000 * (*wst1 - *wst2);
1640 return (double)(wst1->QuadPart - wst2->QuadPart) / 10000;
1644 /* Return the number of milliseconds elapsed since the timer was last
1645 reset. It is allowed to call this function more than once to get
1646 increasingly higher elapsed values. */
1649 wtimer_elapsed (struct wget_timer *wt)
1654 wtimer_sys_set (&now);
1655 elapsed = wt->elapsed_pre_start + wtimer_sys_diff (&now, &wt->start);
1657 /* Ideally we'd just return the difference between NOW and
1658 wt->start. However, the system timer can be set back, and we
1659 could return a value smaller than when we were last called, even
1660 a negative value. Both of these would confuse the callers, which
1661 expect us to return monotonically nondecreasing values.
1663 Therefore: if ELAPSED is smaller than its previous known value,
1664 we reset wt->start to the current time and effectively start
1665 measuring from this point. But since we don't want the elapsed
1666 value to start from zero, we set elapsed_pre_start to the last
1667 elapsed time and increment all future calculations by that
1670 if (elapsed < wt->elapsed_last)
1673 wt->elapsed_pre_start = wt->elapsed_last;
1674 elapsed = wt->elapsed_last;
1677 wt->elapsed_last = elapsed;
1681 /* Return the assessed granularity of the timer implementation, in
1682 milliseconds. This is important for certain code that tries to
1683 deal with "zero" time intervals. */
1686 wtimer_granularity (void)
1688 #ifdef TIMER_GETTIMEOFDAY
1689 /* Granularity of gettimeofday is hugely architecture-dependent.
1690 However, it appears that on modern machines it is better than
1691 1ms. Assume 100 usecs. */
1696 /* This is clear. */
1700 #ifdef TIMER_WINDOWS
1701 /* #### Fill this in! */
1706 /* This should probably be at a better place, but it doesn't really
1707 fit into html-parse.c. */
1709 /* The function returns the pointer to the malloc-ed quoted version of
1710 string s. It will recognize and quote numeric and special graphic
1711 entities, as per RFC1866:
1719 No other entities are recognized or replaced. */
1721 html_quote_string (const char *s)
1727 /* Pass through the string, and count the new size. */
1728 for (i = 0; *s; s++, i++)
1731 i += 4; /* `amp;' */
1732 else if (*s == '<' || *s == '>')
1733 i += 3; /* `lt;' and `gt;' */
1734 else if (*s == '\"')
1735 i += 5; /* `quot;' */
1739 res = (char *)xmalloc (i + 1);
1741 for (p = res; *s; s++)
1754 *p++ = (*s == '<' ? 'l' : 'g');
1781 /* Determine the width of the terminal we're running on. If that's
1782 not possible, return 0. */
1785 determine_screen_width (void)
1787 /* If there's a way to get the terminal size using POSIX
1788 tcgetattr(), somebody please tell me. */
1791 #else /* TIOCGWINSZ */
1795 if (opt.lfilename != NULL)
1798 fd = fileno (stderr);
1799 if (ioctl (fd, TIOCGWINSZ, &wsz) < 0)
1800 return 0; /* most likely ENOTTY */
1803 #endif /* TIOCGWINSZ */
1806 /* Return a random number between 0 and MAX-1, inclusive.
1808 If MAX is greater than the value of RAND_MAX+1 on the system, the
1809 returned value will be in the range [0, RAND_MAX]. This may be
1810 fixed in a future release.
1812 The random number generator is seeded automatically the first time
1815 This uses rand() for portability. It has been suggested that
1816 random() offers better randomness, but this is not required for
1817 Wget, so I chose to go for simplicity and use rand
1821 random_number (int max)
1829 srand (time (NULL));
1834 /* On systems that don't define RAND_MAX, assume it to be 2**15 - 1,
1835 and enforce that assumption by masking other bits. */
1837 # define RAND_MAX 32767
1841 /* This is equivalent to rand() % max, but uses the high-order bits
1842 for better randomness on architecture where rand() is implemented
1843 using a simple congruential generator. */
1845 bounded = (double)max * rnd / (RAND_MAX + 1.0);
1846 return (int)bounded;
1850 /* A debugging function for checking whether an MD5 library works. */
1852 #include "gen-md5.h"
1855 debug_test_md5 (char *buf)
1857 unsigned char raw[16];
1858 static char res[33];
1862 ALLOCA_MD5_CONTEXT (ctx);
1865 gen_md5_update ((unsigned char *)buf, strlen (buf), ctx);
1866 gen_md5_finish (ctx, raw);
1873 *p2++ = XDIGIT_TO_xchar (*p1 >> 4);
1874 *p2++ = XDIGIT_TO_xchar (*p1 & 0xf);
1883 /* Implementation of run_with_timeout, a generic timeout handler for
1884 systems with Unix-like signal handling. */
1885 #ifdef USE_SIGNAL_TIMEOUT
1886 # ifdef HAVE_SIGSETJMP
1887 # define SETJMP(env) sigsetjmp (env, 1)
1889 static sigjmp_buf run_with_timeout_env;
1892 abort_run_with_timeout (int sig)
1894 assert (sig == SIGALRM);
1895 siglongjmp (run_with_timeout_env, -1);
1897 # else /* not HAVE_SIGSETJMP */
1898 # define SETJMP(env) setjmp (env)
1900 static jmp_buf run_with_timeout_env;
1903 abort_run_with_timeout (int sig)
1905 assert (sig == SIGALRM);
1906 /* We don't have siglongjmp to preserve the set of blocked signals;
1907 if we longjumped out of the handler at this point, SIGALRM would
1908 remain blocked. We must unblock it manually. */
1909 int mask = siggetmask ();
1910 mask &= ~sigmask(SIGALRM);
1913 /* Now it's safe to longjump. */
1914 longjmp (run_with_timeout_env, -1);
1916 # endif /* not HAVE_SIGSETJMP */
1917 #endif /* USE_SIGNAL_TIMEOUT */
1920 run_with_timeout (long timeout, void (*fun) (void *), void *arg)
1922 #ifndef USE_SIGNAL_TIMEOUT
1934 signal (SIGALRM, abort_run_with_timeout);
1935 if (SETJMP (run_with_timeout_env) != 0)
1937 /* Longjumped out of FUN with a timeout. */
1938 signal (SIGALRM, SIG_DFL);
1944 /* Preserve errno in case alarm() or signal() modifies it. */
1945 saved_errno = errno;
1947 signal (SIGALRM, SIG_DFL);
1948 errno = saved_errno;