1 /* Various functions of utilitarian nature.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
27 #else /* not HAVE_STRING_H */
29 #endif /* not HAVE_STRING_H */
30 #include <sys/types.h>
35 # include <sys/mman.h>
44 #ifdef HAVE_SYS_UTIME_H
45 # include <sys/utime.h>
49 # include <libc.h> /* for access() */
54 /* For TIOCGWINSZ and friends: */
55 #ifdef HAVE_SYS_IOCTL_H
56 # include <sys/ioctl.h>
62 /* Needed for run_with_timeout. */
63 #undef USE_SIGNAL_TIMEOUT
70 /* If sigsetjmp is a macro, configure won't pick it up. */
72 # define HAVE_SIGSETJMP
75 # ifdef HAVE_SIGSETJMP
76 # define USE_SIGNAL_TIMEOUT
79 # define USE_SIGNAL_TIMEOUT
92 /* This section implements several wrappers around the basic
93 allocation routines. This is done for two reasons: first, so that
94 the callers of these functions need not consistently check for
95 errors. If there is not enough virtual memory for running Wget,
96 something is seriously wrong, and Wget exits with an appropriate
99 The second reason why these are useful is that, if DEBUG_MALLOC is
100 defined, they also provide a handy (if crude) malloc debugging
101 interface that checks memory leaks. */
103 /* Croak the fatal memory error and bail out with non-zero exit
106 memfatal (const char *what)
108 /* Make sure we don't try to store part of the log line, and thus
110 log_set_save_context (0);
111 logprintf (LOG_ALWAYS, _("%s: %s: Not enough memory.\n"), exec_name, what);
115 /* These functions end with _real because they need to be
116 distinguished from the debugging functions, and from the macros.
119 If memory debugging is not turned on, wget.h defines these:
121 #define xmalloc xmalloc_real
122 #define xrealloc xrealloc_real
123 #define xstrdup xstrdup_real
126 In case of memory debugging, the definitions are a bit more
127 complex, because we want to provide more information, *and* we want
128 to call the debugging code. (The former is the reason why xmalloc
129 and friends need to be macros in the first place.) Then it looks
132 #define xmalloc(a) xmalloc_debug (a, __FILE__, __LINE__)
133 #define xfree(a) xfree_debug (a, __FILE__, __LINE__)
134 #define xrealloc(a, b) xrealloc_debug (a, b, __FILE__, __LINE__)
135 #define xstrdup(a) xstrdup_debug (a, __FILE__, __LINE__)
137 Each of the *_debug function does its magic and calls the real one. */
140 # define STATIC_IF_DEBUG static
142 # define STATIC_IF_DEBUG
145 STATIC_IF_DEBUG void *
146 xmalloc_real (size_t size)
148 void *ptr = malloc (size);
154 STATIC_IF_DEBUG void *
155 xrealloc_real (void *ptr, size_t newsize)
159 /* Not all Un*xes have the feature of realloc() that calling it with
160 a NULL-pointer is the same as malloc(), but it is easy to
163 newptr = realloc (ptr, newsize);
165 newptr = malloc (newsize);
167 memfatal ("realloc");
171 STATIC_IF_DEBUG char *
172 xstrdup_real (const char *s)
178 copy = malloc (l + 1);
181 memcpy (copy, s, l + 1);
182 #else /* HAVE_STRDUP */
186 #endif /* HAVE_STRDUP */
193 /* Crude home-grown routines for debugging some malloc-related
196 * Counting the number of malloc and free invocations, and reporting
197 the "balance", i.e. how many times more malloc was called than it
198 was the case with free.
200 * Making malloc store its entry into a simple array and free remove
201 stuff from that array. At the end, print the pointers which have
202 not been freed, along with the source file and the line number.
203 This also has the side-effect of detecting freeing memory that
206 Note that this kind of memory leak checking strongly depends on
207 every malloc() being followed by a free(), even if the program is
208 about to finish. Wget is careful to free the data structure it
209 allocated in init.c. */
211 static int malloc_count, free_count;
217 } malloc_debug[100000];
219 /* Both register_ptr and unregister_ptr take O(n) operations to run,
220 which can be a real problem. It would be nice to use a hash table
221 for malloc_debug, but the functions in hash.c are not suitable
222 because they can call malloc() themselves. Maybe it would work if
223 the hash table were preallocated to a huge size, and if we set the
224 rehash threshold to 1.0. */
226 /* Register PTR in malloc_debug. Abort if this is not possible
227 (presumably due to the number of current allocations exceeding the
228 size of malloc_debug.) */
231 register_ptr (void *ptr, const char *file, int line)
234 for (i = 0; i < ARRAY_SIZE (malloc_debug); i++)
235 if (malloc_debug[i].ptr == NULL)
237 malloc_debug[i].ptr = ptr;
238 malloc_debug[i].file = file;
239 malloc_debug[i].line = line;
245 /* Unregister PTR from malloc_debug. Abort if PTR is not present in
246 malloc_debug. (This catches calling free() with a bogus pointer.) */
249 unregister_ptr (void *ptr)
252 for (i = 0; i < ARRAY_SIZE (malloc_debug); i++)
253 if (malloc_debug[i].ptr == ptr)
255 malloc_debug[i].ptr = NULL;
261 /* Print the malloc debug stats that can be gathered from the above
262 information. Currently this is the count of mallocs, frees, the
263 difference between the two, and the dump of the contents of
264 malloc_debug. The last part are the memory leaks. */
267 print_malloc_debug_stats (void)
270 printf ("\nMalloc: %d\nFree: %d\nBalance: %d\n\n",
271 malloc_count, free_count, malloc_count - free_count);
272 for (i = 0; i < ARRAY_SIZE (malloc_debug); i++)
273 if (malloc_debug[i].ptr != NULL)
274 printf ("0x%08ld: %s:%d\n", (long)malloc_debug[i].ptr,
275 malloc_debug[i].file, malloc_debug[i].line);
279 xmalloc_debug (size_t size, const char *source_file, int source_line)
281 void *ptr = xmalloc_real (size);
283 register_ptr (ptr, source_file, source_line);
288 xfree_debug (void *ptr, const char *source_file, int source_line)
290 assert (ptr != NULL);
292 unregister_ptr (ptr);
297 xrealloc_debug (void *ptr, size_t newsize, const char *source_file, int source_line)
299 void *newptr = xrealloc_real (ptr, newsize);
303 register_ptr (newptr, source_file, source_line);
305 else if (newptr != ptr)
307 unregister_ptr (ptr);
308 register_ptr (newptr, source_file, source_line);
314 xstrdup_debug (const char *s, const char *source_file, int source_line)
316 char *copy = xstrdup_real (s);
318 register_ptr (copy, source_file, source_line);
322 #endif /* DEBUG_MALLOC */
324 /* Utility function: like xstrdup(), but also lowercases S. */
327 xstrdup_lower (const char *s)
329 char *copy = xstrdup (s);
336 /* Return a count of how many times CHR occurs in STRING. */
339 count_char (const char *string, char chr)
343 for (p = string; *p; p++)
349 /* Copy the string formed by two pointers (one on the beginning, other
350 on the char after the last char) to a new, malloc-ed location.
353 strdupdelim (const char *beg, const char *end)
355 char *res = (char *)xmalloc (end - beg + 1);
356 memcpy (res, beg, end - beg);
357 res[end - beg] = '\0';
361 /* Parse a string containing comma-separated elements, and return a
362 vector of char pointers with the elements. Spaces following the
363 commas are ignored. */
365 sepstring (const char *s)
379 res = (char **)xrealloc (res, (i + 2) * sizeof (char *));
380 res[i] = strdupdelim (p, s);
383 /* Skip the blanks following the ','. */
391 res = (char **)xrealloc (res, (i + 2) * sizeof (char *));
392 res[i] = strdupdelim (p, s);
397 /* Return pointer to a static char[] buffer in which zero-terminated
398 string-representation of TM (in form hh:mm:ss) is printed.
400 If TM is non-NULL, the current time-in-seconds will be stored
403 (#### This is misleading: one would expect TM would be used instead
404 of the current time in that case. This design was probably
405 influenced by the design time(2), and should be changed at some
406 points. No callers use non-NULL TM anyway.) */
409 time_str (time_t *tm)
411 static char output[15];
413 time_t secs = time (tm);
417 /* In case of error, return the empty string. Maybe we should
418 just abort if this happens? */
422 ptm = localtime (&secs);
423 sprintf (output, "%02d:%02d:%02d", ptm->tm_hour, ptm->tm_min, ptm->tm_sec);
427 /* Like the above, but include the date: YYYY-MM-DD hh:mm:ss. */
430 datetime_str (time_t *tm)
432 static char output[20]; /* "YYYY-MM-DD hh:mm:ss" + \0 */
434 time_t secs = time (tm);
438 /* In case of error, return the empty string. Maybe we should
439 just abort if this happens? */
443 ptm = localtime (&secs);
444 sprintf (output, "%04d-%02d-%02d %02d:%02d:%02d",
445 ptm->tm_year + 1900, ptm->tm_mon + 1, ptm->tm_mday,
446 ptm->tm_hour, ptm->tm_min, ptm->tm_sec);
450 /* The Windows versions of the following two functions are defined in
455 fork_to_background (void)
458 /* Whether we arrange our own version of opt.lfilename here. */
463 opt.lfilename = unique_name (DEFAULT_LOGFILE);
475 /* parent, no error */
476 printf (_("Continuing in background, pid %d.\n"), (int)pid);
478 printf (_("Output will be written to `%s'.\n"), opt.lfilename);
479 exit (0); /* #### should we use _exit()? */
482 /* child: give up the privileges and keep running. */
484 freopen ("/dev/null", "r", stdin);
485 freopen ("/dev/null", "w", stdout);
486 freopen ("/dev/null", "w", stderr);
488 #endif /* not WINDOWS */
490 /* "Touch" FILE, i.e. make its atime and mtime equal to the time
491 specified with TM. */
493 touch (const char *file, time_t tm)
495 #ifdef HAVE_STRUCT_UTIMBUF
496 struct utimbuf times;
497 times.actime = times.modtime = tm;
500 times[0] = times[1] = tm;
503 if (utime (file, ×) == -1)
504 logprintf (LOG_NOTQUIET, "utime(%s): %s\n", file, strerror (errno));
507 /* Checks if FILE is a symbolic link, and removes it if it is. Does
508 nothing under MS-Windows. */
510 remove_link (const char *file)
515 if (lstat (file, &st) == 0 && S_ISLNK (st.st_mode))
517 DEBUGP (("Unlinking %s (symlink).\n", file));
520 logprintf (LOG_VERBOSE, _("Failed to unlink symlink `%s': %s\n"),
521 file, strerror (errno));
526 /* Does FILENAME exist? This is quite a lousy implementation, since
527 it supplies no error codes -- only a yes-or-no answer. Thus it
528 will return that a file does not exist if, e.g., the directory is
529 unreadable. I don't mind it too much currently, though. The
530 proper way should, of course, be to have a third, error state,
531 other than true/false, but that would introduce uncalled-for
532 additional complexity to the callers. */
534 file_exists_p (const char *filename)
537 return access (filename, F_OK) >= 0;
540 return stat (filename, &buf) >= 0;
544 /* Returns 0 if PATH is a directory, 1 otherwise (any kind of file).
545 Returns 0 on error. */
547 file_non_directory_p (const char *path)
550 /* Use lstat() rather than stat() so that symbolic links pointing to
551 directories can be identified correctly. */
552 if (lstat (path, &buf) != 0)
554 return S_ISDIR (buf.st_mode) ? 0 : 1;
557 /* Return the size of file named by FILENAME, or -1 if it cannot be
558 opened or seeked into. */
560 file_size (const char *filename)
563 /* We use fseek rather than stat to determine the file size because
564 that way we can also verify whether the file is readable.
565 Inspired by the POST patch by Arnaud Wylie. */
566 FILE *fp = fopen (filename, "rb");
567 fseek (fp, 0, SEEK_END);
573 /* Return a unique filename, given a prefix and count */
575 unique_name_1 (const char *fileprefix, int count)
581 filename = (char *)xmalloc (strlen (fileprefix) + numdigit (count) + 2);
582 sprintf (filename, "%s.%d", fileprefix, count);
585 filename = xstrdup (fileprefix);
587 if (!file_exists_p (filename))
596 /* Return a unique file name, based on PREFIX. */
598 unique_name (const char *prefix)
604 file = unique_name_1 (prefix, count++);
608 /* Create DIRECTORY. If some of the pathname components of DIRECTORY
609 are missing, create them first. In case any mkdir() call fails,
610 return its error status. Returns 0 on successful completion.
612 The behaviour of this function should be identical to the behaviour
613 of `mkdir -p' on systems where mkdir supports the `-p' option. */
615 make_directory (const char *directory)
622 /* Make a copy of dir, to be able to write to it. Otherwise, the
623 function is unsafe if called with a read-only char *argument. */
624 STRDUP_ALLOCA (dir, directory);
626 /* If the first character of dir is '/', skip it (and thus enable
627 creation of absolute-pathname directories. */
628 for (i = (*dir == '/'); 1; ++i)
630 for (; dir[i] && dir[i] != '/'; i++)
635 /* Check whether the directory already exists. Allow creation of
636 of intermediate directories to fail, as the initial path components
637 are not necessarily directories! */
638 if (!file_exists_p (dir))
639 ret = mkdir (dir, 0777);
650 /* Merge BASE with FILE. BASE can be a directory or a file name, FILE
651 should be a file name.
653 file_merge("/foo/bar", "baz") => "/foo/baz"
654 file_merge("/foo/bar/", "baz") => "/foo/bar/baz"
655 file_merge("foo", "bar") => "bar"
657 In other words, it's a simpler and gentler version of uri_merge_1. */
660 file_merge (const char *base, const char *file)
663 const char *cut = (const char *)strrchr (base, '/');
666 return xstrdup (file);
668 result = (char *)xmalloc (cut - base + 1 + strlen (file) + 1);
669 memcpy (result, base, cut - base);
670 result[cut - base] = '/';
671 strcpy (result + (cut - base) + 1, file);
676 static int in_acclist PARAMS ((const char *const *, const char *, int));
678 /* Determine whether a file is acceptable to be followed, according to
679 lists of patterns to accept/reject. */
681 acceptable (const char *s)
685 while (l && s[l] != '/')
692 return (in_acclist ((const char *const *)opt.accepts, s, 1)
693 && !in_acclist ((const char *const *)opt.rejects, s, 1));
695 return in_acclist ((const char *const *)opt.accepts, s, 1);
697 else if (opt.rejects)
698 return !in_acclist ((const char *const *)opt.rejects, s, 1);
702 /* Compare S1 and S2 frontally; S2 must begin with S1. E.g. if S1 is
703 `/something', frontcmp() will return 1 only if S2 begins with
704 `/something'. Otherwise, 0 is returned. */
706 frontcmp (const char *s1, const char *s2)
708 for (; *s1 && *s2 && (*s1 == *s2); ++s1, ++s2);
712 /* Iterate through STRLIST, and return the first element that matches
713 S, through wildcards or front comparison (as appropriate). */
715 proclist (char **strlist, const char *s, enum accd flags)
719 for (x = strlist; *x; x++)
720 if (has_wildcards_p (*x))
722 if (fnmatch (*x, s, FNM_PATHNAME) == 0)
727 char *p = *x + ((flags & ALLABS) && (**x == '/')); /* Remove '/' */
734 /* Returns whether DIRECTORY is acceptable for download, wrt the
735 include/exclude lists.
737 If FLAGS is ALLABS, the leading `/' is ignored in paths; relative
738 and absolute paths may be freely intermixed. */
740 accdir (const char *directory, enum accd flags)
742 /* Remove starting '/'. */
743 if (flags & ALLABS && *directory == '/')
747 if (!proclist (opt.includes, directory, flags))
752 if (proclist (opt.excludes, directory, flags))
758 /* Match the end of STRING against PATTERN. For instance:
760 match_backwards ("abc", "bc") -> 1
761 match_backwards ("abc", "ab") -> 0
762 match_backwards ("abc", "abc") -> 1 */
764 match_tail (const char *string, const char *pattern)
768 for (i = strlen (string), j = strlen (pattern); i >= 0 && j >= 0; i--, j--)
769 if (string[i] != pattern[j])
771 /* If the pattern was exhausted, the match was succesful. */
778 /* Checks whether string S matches each element of ACCEPTS. A list
779 element are matched either with fnmatch() or match_tail(),
780 according to whether the element contains wildcards or not.
782 If the BACKWARD is 0, don't do backward comparison -- just compare
785 in_acclist (const char *const *accepts, const char *s, int backward)
787 for (; *accepts; accepts++)
789 if (has_wildcards_p (*accepts))
791 /* fnmatch returns 0 if the pattern *does* match the
793 if (fnmatch (*accepts, s, 0) == 0)
800 if (match_tail (s, *accepts))
805 if (!strcmp (s, *accepts))
813 /* Return the location of STR's suffix (file extension). Examples:
814 suffix ("foo.bar") -> "bar"
815 suffix ("foo.bar.baz") -> "baz"
816 suffix ("/foo/bar") -> NULL
817 suffix ("/foo.bar/baz") -> NULL */
819 suffix (const char *str)
823 for (i = strlen (str); i && str[i] != '/' && str[i] != '.'; i--)
827 return (char *)str + i;
832 /* Return non-zero if FNAME ends with a typical HTML suffix. The
833 following (case-insensitive) suffixes are presumed to be HTML files:
837 ?html (`?' matches one character)
839 #### CAVEAT. This is not necessarily a good indication that FNAME
840 refers to a file that contains HTML! */
842 has_html_suffix_p (const char *fname)
846 if ((suf = suffix (fname)) == NULL)
848 if (!strcasecmp (suf, "html"))
850 if (!strcasecmp (suf, "htm"))
852 if (suf[0] && !strcasecmp (suf + 1, "html"))
857 /* Read a line from FP and return the pointer to freshly allocated
858 storage. The stoarage space is obtained through malloc() and
859 should be freed with free() when it is no longer needed.
861 The length of the line is not limited, except by available memory.
862 The newline character at the end of line is retained. The line is
863 terminated with a zero character.
865 After end-of-file is encountered without anything being read, NULL
866 is returned. NULL is also returned on error. To distinguish
867 between these two cases, use the stdio function ferror(). */
870 read_whole_line (FILE *fp)
874 char *line = (char *)xmalloc (bufsize);
876 while (fgets (line + length, bufsize - length, fp))
878 length += strlen (line + length);
880 /* Possible for example when reading from a binary file where
881 a line begins with \0. */
884 if (line[length - 1] == '\n')
887 /* fgets() guarantees to read the whole line, or to use up the
888 space we've given it. We can double the buffer
891 line = xrealloc (line, bufsize);
893 if (length == 0 || ferror (fp))
898 if (length + 1 < bufsize)
899 /* Relieve the memory from our exponential greediness. We say
900 `length + 1' because the terminating \0 is not included in
901 LENGTH. We don't need to zero-terminate the string ourselves,
902 though, because fgets() does that. */
903 line = xrealloc (line, length + 1);
907 /* Read FILE into memory. A pointer to `struct file_memory' are
908 returned; use struct element `content' to access file contents, and
909 the element `length' to know the file length. `content' is *not*
910 zero-terminated, and you should *not* read or write beyond the [0,
911 length) range of characters.
913 After you are done with the file contents, call read_file_free to
916 Depending on the operating system and the type of file that is
917 being read, read_file() either mmap's the file into memory, or
918 reads the file into the core using read().
920 If file is named "-", fileno(stdin) is used for reading instead.
921 If you want to read from a real file named "-", use "./-" instead. */
924 read_file (const char *file)
927 struct file_memory *fm;
929 int inhibit_close = 0;
931 /* Some magic in the finest tradition of Perl and its kin: if FILE
932 is "-", just use stdin. */
937 /* Note that we don't inhibit mmap() in this case. If stdin is
938 redirected from a regular file, mmap() will still work. */
941 fd = open (file, O_RDONLY);
944 fm = xmalloc (sizeof (struct file_memory));
949 if (fstat (fd, &buf) < 0)
951 fm->length = buf.st_size;
952 /* NOTE: As far as I know, the callers of this function never
953 modify the file text. Relying on this would enable us to
954 specify PROT_READ and MAP_SHARED for a marginal gain in
955 efficiency, but at some cost to generality. */
956 fm->content = mmap (NULL, fm->length, PROT_READ | PROT_WRITE,
958 if (fm->content == (char *)MAP_FAILED)
968 /* The most common reason why mmap() fails is that FD does not point
969 to a plain file. However, it's also possible that mmap() doesn't
970 work for a particular type of file. Therefore, whenever mmap()
971 fails, we just fall back to the regular method. */
972 #endif /* HAVE_MMAP */
975 size = 512; /* number of bytes fm->contents can
976 hold at any given time. */
977 fm->content = xmalloc (size);
981 if (fm->length > size / 2)
983 /* #### I'm not sure whether the whole exponential-growth
984 thing makes sense with kernel read. On Linux at least,
985 read() refuses to read more than 4K from a file at a
986 single chunk anyway. But other Unixes might optimize it
987 better, and it doesn't *hurt* anything, so I'm leaving
990 /* Normally, we grow SIZE exponentially to make the number
991 of calls to read() and realloc() logarithmic in relation
992 to file size. However, read() can read an amount of data
993 smaller than requested, and it would be unreasonably to
994 double SIZE every time *something* was read. Therefore,
995 we double SIZE only when the length exceeds half of the
996 entire allocated size. */
998 fm->content = xrealloc (fm->content, size);
1000 nread = read (fd, fm->content + fm->length, size - fm->length);
1002 /* Successful read. */
1003 fm->length += nread;
1013 if (size > fm->length && fm->length != 0)
1014 /* Due to exponential growth of fm->content, the allocated region
1015 might be much larger than what is actually needed. */
1016 fm->content = xrealloc (fm->content, fm->length);
1023 xfree (fm->content);
1028 /* Release the resources held by FM. Specifically, this calls
1029 munmap() or xfree() on fm->content, depending whether mmap or
1030 malloc/read were used to read in the file. It also frees the
1031 memory needed to hold the FM structure itself. */
1034 read_file_free (struct file_memory *fm)
1039 munmap (fm->content, fm->length);
1044 xfree (fm->content);
1049 /* Free the pointers in a NULL-terminated vector of pointers, then
1050 free the pointer itself. */
1052 free_vec (char **vec)
1063 /* Append vector V2 to vector V1. The function frees V2 and
1064 reallocates V1 (thus you may not use the contents of neither
1065 pointer after the call). If V1 is NULL, V2 is returned. */
1067 merge_vecs (char **v1, char **v2)
1077 /* To avoid j == 0 */
1082 for (i = 0; v1[i]; i++);
1084 for (j = 0; v2[j]; j++);
1085 /* Reallocate v1. */
1086 v1 = (char **)xrealloc (v1, (i + j + 1) * sizeof (char **));
1087 memcpy (v1 + i, v2, (j + 1) * sizeof (char *));
1092 /* A set of simple-minded routines to store strings in a linked list.
1093 This used to also be used for searching, but now we have hash
1096 /* It's a shame that these simple things like linked lists and hash
1097 tables (see hash.c) need to be implemented over and over again. It
1098 would be nice to be able to use the routines from glib -- see
1099 www.gtk.org for details. However, that would make Wget depend on
1100 glib, and I want to avoid dependencies to external libraries for
1101 reasons of convenience and portability (I suspect Wget is more
1102 portable than anything ever written for Gnome). */
1104 /* Append an element to the list. If the list has a huge number of
1105 elements, this can get slow because it has to find the list's
1106 ending. If you think you have to call slist_append in a loop,
1107 think about calling slist_prepend() followed by slist_nreverse(). */
1110 slist_append (slist *l, const char *s)
1112 slist *newel = (slist *)xmalloc (sizeof (slist));
1115 newel->string = xstrdup (s);
1120 /* Find the last element. */
1127 /* Prepend S to the list. Unlike slist_append(), this is O(1). */
1130 slist_prepend (slist *l, const char *s)
1132 slist *newel = (slist *)xmalloc (sizeof (slist));
1133 newel->string = xstrdup (s);
1138 /* Destructively reverse L. */
1141 slist_nreverse (slist *l)
1146 slist *next = l->next;
1154 /* Is there a specific entry in the list? */
1156 slist_contains (slist *l, const char *s)
1158 for (; l; l = l->next)
1159 if (!strcmp (l->string, s))
1164 /* Free the whole slist. */
1166 slist_free (slist *l)
1177 /* Sometimes it's useful to create "sets" of strings, i.e. special
1178 hash tables where you want to store strings as keys and merely
1179 query for their existence. Here is a set of utility routines that
1180 makes that transparent. */
1183 string_set_add (struct hash_table *ht, const char *s)
1185 /* First check whether the set element already exists. If it does,
1186 do nothing so that we don't have to free() the old element and
1187 then strdup() a new one. */
1188 if (hash_table_contains (ht, s))
1191 /* We use "1" as value. It provides us a useful and clear arbitrary
1192 value, and it consumes no memory -- the pointers to the same
1193 string "1" will be shared by all the key-value pairs in all `set'
1195 hash_table_put (ht, xstrdup (s), "1");
1198 /* Synonym for hash_table_contains... */
1201 string_set_contains (struct hash_table *ht, const char *s)
1203 return hash_table_contains (ht, s);
1207 string_set_free_mapper (void *key, void *value_ignored, void *arg_ignored)
1214 string_set_free (struct hash_table *ht)
1216 hash_table_map (ht, string_set_free_mapper, NULL);
1217 hash_table_destroy (ht);
1221 free_keys_and_values_mapper (void *key, void *value, void *arg_ignored)
1228 /* Another utility function: call free() on all keys and values of HT. */
1231 free_keys_and_values (struct hash_table *ht)
1233 hash_table_map (ht, free_keys_and_values_mapper, NULL);
1237 /* Engine for legible and legible_very_long; this function works on
1241 legible_1 (const char *repr)
1243 static char outbuf[128];
1248 /* Reset the pointers. */
1251 /* If the number is negative, shift the pointers. */
1257 /* How many digits before the first separator? */
1258 mod = strlen (inptr) % 3;
1260 for (i = 0; i < mod; i++)
1261 *outptr++ = inptr[i];
1262 /* Now insert the rest of them, putting separator before every
1264 for (i1 = i, i = 0; inptr[i1]; i++, i1++)
1266 if (i % 3 == 0 && i1 != 0)
1268 *outptr++ = inptr[i1];
1270 /* Zero-terminate the string. */
1275 /* Legible -- return a static pointer to the legibly printed long. */
1280 /* Print the number into the buffer. */
1281 number_to_string (inbuf, l);
1282 return legible_1 (inbuf);
1285 /* Write a string representation of NUMBER into the provided buffer.
1286 We cannot use sprintf() because we cannot be sure whether the
1287 platform supports printing of what we chose for VERY_LONG_TYPE.
1289 Example: Gcc supports `long long' under many platforms, but on many
1290 of those the native libc knows nothing of it and therefore cannot
1293 How long BUFFER needs to be depends on the platform and the content
1294 of NUMBER. For 64-bit VERY_LONG_TYPE (the most common case), 24
1295 bytes are sufficient. Using more might be a good idea.
1297 This function does not go through the hoops that long_to_string
1298 goes to because it doesn't aspire to be fast. (It's called perhaps
1299 once in a Wget run.) */
1302 very_long_to_string (char *buffer, VERY_LONG_TYPE number)
1307 /* Print the number backwards... */
1310 buffer[i++] = '0' + number % 10;
1315 /* ...and reverse the order of the digits. */
1316 for (j = 0; j < i / 2; j++)
1319 buffer[j] = buffer[i - 1 - j];
1320 buffer[i - 1 - j] = c;
1325 /* The same as legible(), but works on VERY_LONG_TYPE. See sysdep.h. */
1327 legible_very_long (VERY_LONG_TYPE l)
1330 /* Print the number into the buffer. */
1331 very_long_to_string (inbuf, l);
1332 return legible_1 (inbuf);
1335 /* Count the digits in a (long) integer. */
1337 numdigit (long number)
1345 while ((number /= 10) > 0)
1350 #define ONE_DIGIT(figure) *p++ = n / (figure) + '0'
1351 #define ONE_DIGIT_ADVANCE(figure) (ONE_DIGIT (figure), n %= (figure))
1353 #define DIGITS_1(figure) ONE_DIGIT (figure)
1354 #define DIGITS_2(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_1 ((figure) / 10)
1355 #define DIGITS_3(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_2 ((figure) / 10)
1356 #define DIGITS_4(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_3 ((figure) / 10)
1357 #define DIGITS_5(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_4 ((figure) / 10)
1358 #define DIGITS_6(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_5 ((figure) / 10)
1359 #define DIGITS_7(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_6 ((figure) / 10)
1360 #define DIGITS_8(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_7 ((figure) / 10)
1361 #define DIGITS_9(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_8 ((figure) / 10)
1362 #define DIGITS_10(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_9 ((figure) / 10)
1364 /* DIGITS_<11-20> are only used on machines with 64-bit longs. */
1366 #define DIGITS_11(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_10 ((figure) / 10)
1367 #define DIGITS_12(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_11 ((figure) / 10)
1368 #define DIGITS_13(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_12 ((figure) / 10)
1369 #define DIGITS_14(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_13 ((figure) / 10)
1370 #define DIGITS_15(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_14 ((figure) / 10)
1371 #define DIGITS_16(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_15 ((figure) / 10)
1372 #define DIGITS_17(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_16 ((figure) / 10)
1373 #define DIGITS_18(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_17 ((figure) / 10)
1374 #define DIGITS_19(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_18 ((figure) / 10)
1376 /* Print NUMBER to BUFFER in base 10. This should be completely
1377 equivalent to `sprintf(buffer, "%ld", number)', only much faster.
1379 The speedup may make a difference in programs that frequently
1380 convert numbers to strings. Some implementations of sprintf,
1381 particularly the one in GNU libc, have been known to be extremely
1382 slow compared to this function.
1384 Return the pointer to the location where the terminating zero was
1385 printed. (Equivalent to calling buffer+strlen(buffer) after the
1388 BUFFER should be big enough to accept as many bytes as you expect
1389 the number to take up. On machines with 64-bit longs the maximum
1390 needed size is 24 bytes. That includes the digits needed for the
1391 largest 64-bit number, the `-' sign in case it's negative, and the
1392 terminating '\0'. */
1395 number_to_string (char *buffer, long number)
1400 #if (SIZEOF_LONG != 4) && (SIZEOF_LONG != 8)
1401 /* We are running in a strange or misconfigured environment. Let
1402 sprintf cope with it. */
1403 sprintf (buffer, "%ld", n);
1404 p += strlen (buffer);
1405 #else /* (SIZEOF_LONG == 4) || (SIZEOF_LONG == 8) */
1413 if (n < 10) { DIGITS_1 (1); }
1414 else if (n < 100) { DIGITS_2 (10); }
1415 else if (n < 1000) { DIGITS_3 (100); }
1416 else if (n < 10000) { DIGITS_4 (1000); }
1417 else if (n < 100000) { DIGITS_5 (10000); }
1418 else if (n < 1000000) { DIGITS_6 (100000); }
1419 else if (n < 10000000) { DIGITS_7 (1000000); }
1420 else if (n < 100000000) { DIGITS_8 (10000000); }
1421 else if (n < 1000000000) { DIGITS_9 (100000000); }
1422 #if SIZEOF_LONG == 4
1423 /* ``if (1)'' serves only to preserve editor indentation. */
1424 else if (1) { DIGITS_10 (1000000000); }
1425 #else /* SIZEOF_LONG != 4 */
1426 else if (n < 10000000000L) { DIGITS_10 (1000000000L); }
1427 else if (n < 100000000000L) { DIGITS_11 (10000000000L); }
1428 else if (n < 1000000000000L) { DIGITS_12 (100000000000L); }
1429 else if (n < 10000000000000L) { DIGITS_13 (1000000000000L); }
1430 else if (n < 100000000000000L) { DIGITS_14 (10000000000000L); }
1431 else if (n < 1000000000000000L) { DIGITS_15 (100000000000000L); }
1432 else if (n < 10000000000000000L) { DIGITS_16 (1000000000000000L); }
1433 else if (n < 100000000000000000L) { DIGITS_17 (10000000000000000L); }
1434 else if (n < 1000000000000000000L) { DIGITS_18 (100000000000000000L); }
1435 else { DIGITS_19 (1000000000000000000L); }
1436 #endif /* SIZEOF_LONG != 4 */
1439 #endif /* (SIZEOF_LONG == 4) || (SIZEOF_LONG == 8) */
1445 #undef ONE_DIGIT_ADVANCE
1467 /* Support for timers. */
1469 #undef TIMER_WINDOWS
1470 #undef TIMER_GETTIMEOFDAY
1473 /* Depending on the OS and availability of gettimeofday(), one and
1474 only one of the above constants will be defined. Virtually all
1475 modern Unix systems will define TIMER_GETTIMEOFDAY; Windows will
1476 use TIMER_WINDOWS. TIMER_TIME is a catch-all method for
1477 non-Windows systems without gettimeofday.
1479 #### Perhaps we should also support ftime(), which exists on old
1480 BSD 4.2-influenced systems? (It also existed under MS DOS Borland
1481 C, if memory serves me.) */
1484 # define TIMER_WINDOWS
1485 #else /* not WINDOWS */
1486 # ifdef HAVE_GETTIMEOFDAY
1487 # define TIMER_GETTIMEOFDAY
1491 #endif /* not WINDOWS */
1494 #ifdef TIMER_GETTIMEOFDAY
1503 #ifdef TIMER_WINDOWS
1504 ULARGE_INTEGER wintime;
1508 /* Allocate a timer. It is not legal to do anything with a freshly
1509 allocated timer, except call wtimer_reset() or wtimer_delete(). */
1512 wtimer_allocate (void)
1514 struct wget_timer *wt =
1515 (struct wget_timer *)xmalloc (sizeof (struct wget_timer));
1519 /* Allocate a new timer and reset it. Return the new timer. */
1524 struct wget_timer *wt = wtimer_allocate ();
1529 /* Free the resources associated with the timer. Its further use is
1533 wtimer_delete (struct wget_timer *wt)
1538 /* Reset timer WT. This establishes the starting point from which
1539 wtimer_elapsed() will return the number of elapsed
1540 milliseconds. It is allowed to reset a previously used timer. */
1543 wtimer_reset (struct wget_timer *wt)
1545 #ifdef TIMER_GETTIMEOFDAY
1547 gettimeofday (&t, NULL);
1548 wt->secs = t.tv_sec;
1549 wt->usecs = t.tv_usec;
1553 wt->secs = time (NULL);
1556 #ifdef TIMER_WINDOWS
1559 GetSystemTime (&st);
1560 SystemTimeToFileTime (&st, &ft);
1561 wt->wintime.HighPart = ft.dwHighDateTime;
1562 wt->wintime.LowPart = ft.dwLowDateTime;
1566 /* Return the number of milliseconds elapsed since the timer was last
1567 reset. It is allowed to call this function more than once to get
1568 increasingly higher elapsed values. */
1571 wtimer_elapsed (struct wget_timer *wt)
1573 #ifdef TIMER_GETTIMEOFDAY
1575 gettimeofday (&t, NULL);
1576 return (t.tv_sec - wt->secs) * 1000 + (t.tv_usec - wt->usecs) / 1000;
1580 time_t now = time (NULL);
1581 return 1000 * (now - wt->secs);
1588 GetSystemTime (&st);
1589 SystemTimeToFileTime (&st, &ft);
1590 uli.HighPart = ft.dwHighDateTime;
1591 uli.LowPart = ft.dwLowDateTime;
1592 return (long)((uli.QuadPart - wt->wintime.QuadPart) / 10000);
1596 /* Return the assessed granularity of the timer implementation. This
1597 is important for certain code that tries to deal with "zero" time
1601 wtimer_granularity (void)
1603 #ifdef TIMER_GETTIMEOFDAY
1604 /* Granularity of gettimeofday is hugely architecture-dependent.
1605 However, it appears that on modern machines it is better than
1611 /* This is clear. */
1615 #ifdef TIMER_WINDOWS
1621 /* This should probably be at a better place, but it doesn't really
1622 fit into html-parse.c. */
1624 /* The function returns the pointer to the malloc-ed quoted version of
1625 string s. It will recognize and quote numeric and special graphic
1626 entities, as per RFC1866:
1634 No other entities are recognized or replaced. */
1636 html_quote_string (const char *s)
1642 /* Pass through the string, and count the new size. */
1643 for (i = 0; *s; s++, i++)
1646 i += 4; /* `amp;' */
1647 else if (*s == '<' || *s == '>')
1648 i += 3; /* `lt;' and `gt;' */
1649 else if (*s == '\"')
1650 i += 5; /* `quot;' */
1654 res = (char *)xmalloc (i + 1);
1656 for (p = res; *s; s++)
1669 *p++ = (*s == '<' ? 'l' : 'g');
1696 /* Determine the width of the terminal we're running on. If that's
1697 not possible, return 0. */
1700 determine_screen_width (void)
1702 /* If there's a way to get the terminal size using POSIX
1703 tcgetattr(), somebody please tell me. */
1706 #else /* TIOCGWINSZ */
1710 if (opt.lfilename != NULL)
1713 fd = fileno (stderr);
1714 if (ioctl (fd, TIOCGWINSZ, &wsz) < 0)
1715 return 0; /* most likely ENOTTY */
1718 #endif /* TIOCGWINSZ */
1721 /* Return a random number between 0 and MAX-1, inclusive.
1723 If MAX is greater than the value of RAND_MAX+1 on the system, the
1724 returned value will be in the range [0, RAND_MAX]. This may be
1725 fixed in a future release.
1727 The random number generator is seeded automatically the first time
1730 This uses rand() for portability. It has been suggested that
1731 random() offers better randomness, but this is not required for
1732 Wget, so I chose to go for simplicity and use rand
1736 random_number (int max)
1744 srand (time (NULL));
1749 /* On systems that don't define RAND_MAX, assume it to be 2**15 - 1,
1750 and enforce that assumption by masking other bits. */
1752 # define RAND_MAX 32767
1756 /* This is equivalent to rand() % max, but uses the high-order bits
1757 for better randomness on architecture where rand() is implemented
1758 using a simple congruential generator. */
1760 bounded = (double)max * rnd / (RAND_MAX + 1.0);
1761 return (int)bounded;
1765 /* A debugging function for checking whether an MD5 library works. */
1767 #include "gen-md5.h"
1770 debug_test_md5 (char *buf)
1772 unsigned char raw[16];
1773 static char res[33];
1777 ALLOCA_MD5_CONTEXT (ctx);
1780 gen_md5_update ((unsigned char *)buf, strlen (buf), ctx);
1781 gen_md5_finish (ctx, raw);
1788 *p2++ = XDIGIT_TO_xchar (*p1 >> 4);
1789 *p2++ = XDIGIT_TO_xchar (*p1 & 0xf);
1798 /* Implementation of run_with_timeout, a generic timeout handler for
1799 systems with Unix-like signal handling. */
1800 #ifdef HAVE_SIGSETJMP
1801 #define SETJMP(env) sigsetjmp (env, 1)
1803 static sigjmp_buf run_with_timeout_env;
1806 abort_run_with_timeout (int sig)
1808 assert (sig == SIGALRM);
1809 siglongjmp (run_with_timeout_env, -1);
1811 #else /* not HAVE_SIGSETJMP */
1812 #define SETJMP(env) setjmp (env)
1814 static jmp_buf run_with_timeout_env;
1817 abort_run_with_timeout (int sig)
1819 assert (sig == SIGALRM);
1820 /* We don't have siglongjmp to preserve the set of blocked signals;
1821 if we longjumped out of the handler at this point, SIGALRM would
1822 remain blocked. We must unblock it manually. */
1823 int mask = siggetmask ();
1824 mask &= ~sigmask(SIGALRM);
1827 /* Now it's safe to longjump. */
1828 longjmp (run_with_timeout_env, -1);
1830 #endif /* not HAVE_SIGSETJMP */
1833 run_with_timeout (long timeout, void (*fun) (void *), void *arg)
1835 #ifndef USE_SIGNAL_TIMEOUT
1847 signal (SIGALRM, abort_run_with_timeout);
1848 if (SETJMP (run_with_timeout_env) != 0)
1850 /* Longjumped out of FUN with a timeout. */
1851 signal (SIGALRM, SIG_DFL);
1857 /* Preserve errno in case alarm() or signal() modifies it. */
1858 saved_errno = errno;
1860 signal (SIGALRM, SIG_DFL);
1861 errno = saved_errno;