1 /* Various functions of utilitarian nature.
2 Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
3 Free Software Foundation, Inc.
5 This file is part of GNU Wget.
7 GNU Wget is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 GNU Wget is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with Wget; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
27 #else /* not HAVE_STRING_H */
29 #endif /* not HAVE_STRING_H */
30 #include <sys/types.h>
35 # include <sys/mman.h>
44 #ifdef HAVE_SYS_UTIME_H
45 # include <sys/utime.h>
49 # include <libc.h> /* for access() */
54 /* For TIOCGWINSZ and friends: */
55 #ifdef HAVE_SYS_IOCTL_H
56 # include <sys/ioctl.h>
62 /* Needed for run_with_timeout. */
63 #undef USE_SIGNAL_TIMEOUT
70 /* If sigsetjmp is a macro, configure won't pick it up. */
72 # define HAVE_SIGSETJMP
75 # ifdef HAVE_SIGSETJMP
76 # define USE_SIGNAL_TIMEOUT
79 # define USE_SIGNAL_TIMEOUT
92 /* This section implements several wrappers around the basic
93 allocation routines. This is done for two reasons: first, so that
94 the callers of these functions need not consistently check for
95 errors. If there is not enough virtual memory for running Wget,
96 something is seriously wrong, and Wget exits with an appropriate
99 The second reason why these are useful is that, if DEBUG_MALLOC is
100 defined, they also provide a handy (if crude) malloc debugging
101 interface that checks memory leaks. */
103 /* Croak the fatal memory error and bail out with non-zero exit
106 memfatal (const char *what)
108 /* Make sure we don't try to store part of the log line, and thus
110 log_set_save_context (0);
111 logprintf (LOG_ALWAYS, _("%s: %s: Not enough memory.\n"), exec_name, what);
115 /* These functions end with _real because they need to be
116 distinguished from the debugging functions, and from the macros.
119 If memory debugging is not turned on, wget.h defines these:
121 #define xmalloc xmalloc_real
122 #define xrealloc xrealloc_real
123 #define xstrdup xstrdup_real
126 In case of memory debugging, the definitions are a bit more
127 complex, because we want to provide more information, *and* we want
128 to call the debugging code. (The former is the reason why xmalloc
129 and friends need to be macros in the first place.) Then it looks
132 #define xmalloc(a) xmalloc_debug (a, __FILE__, __LINE__)
133 #define xfree(a) xfree_debug (a, __FILE__, __LINE__)
134 #define xrealloc(a, b) xrealloc_debug (a, b, __FILE__, __LINE__)
135 #define xstrdup(a) xstrdup_debug (a, __FILE__, __LINE__)
137 Each of the *_debug function does its magic and calls the real one. */
140 # define STATIC_IF_DEBUG static
142 # define STATIC_IF_DEBUG
145 STATIC_IF_DEBUG void *
146 xmalloc_real (size_t size)
148 void *ptr = malloc (size);
154 STATIC_IF_DEBUG void *
155 xrealloc_real (void *ptr, size_t newsize)
159 /* Not all Un*xes have the feature of realloc() that calling it with
160 a NULL-pointer is the same as malloc(), but it is easy to
163 newptr = realloc (ptr, newsize);
165 newptr = malloc (newsize);
167 memfatal ("realloc");
171 STATIC_IF_DEBUG char *
172 xstrdup_real (const char *s)
178 copy = malloc (l + 1);
181 memcpy (copy, s, l + 1);
182 #else /* HAVE_STRDUP */
186 #endif /* HAVE_STRDUP */
193 /* Crude home-grown routines for debugging some malloc-related
196 * Counting the number of malloc and free invocations, and reporting
197 the "balance", i.e. how many times more malloc was called than it
198 was the case with free.
200 * Making malloc store its entry into a simple array and free remove
201 stuff from that array. At the end, print the pointers which have
202 not been freed, along with the source file and the line number.
203 This also has the side-effect of detecting freeing memory that
206 Note that this kind of memory leak checking strongly depends on
207 every malloc() being followed by a free(), even if the program is
208 about to finish. Wget is careful to free the data structure it
209 allocated in init.c. */
211 static int malloc_count, free_count;
217 } malloc_debug[100000];
219 /* Both register_ptr and unregister_ptr take O(n) operations to run,
220 which can be a real problem. It would be nice to use a hash table
221 for malloc_debug, but the functions in hash.c are not suitable
222 because they can call malloc() themselves. Maybe it would work if
223 the hash table were preallocated to a huge size, and if we set the
224 rehash threshold to 1.0. */
226 /* Register PTR in malloc_debug. Abort if this is not possible
227 (presumably due to the number of current allocations exceeding the
228 size of malloc_debug.) */
231 register_ptr (void *ptr, const char *file, int line)
234 for (i = 0; i < ARRAY_SIZE (malloc_debug); i++)
235 if (malloc_debug[i].ptr == NULL)
237 malloc_debug[i].ptr = ptr;
238 malloc_debug[i].file = file;
239 malloc_debug[i].line = line;
245 /* Unregister PTR from malloc_debug. Abort if PTR is not present in
246 malloc_debug. (This catches calling free() with a bogus pointer.) */
249 unregister_ptr (void *ptr)
252 for (i = 0; i < ARRAY_SIZE (malloc_debug); i++)
253 if (malloc_debug[i].ptr == ptr)
255 malloc_debug[i].ptr = NULL;
261 /* Print the malloc debug stats that can be gathered from the above
262 information. Currently this is the count of mallocs, frees, the
263 difference between the two, and the dump of the contents of
264 malloc_debug. The last part are the memory leaks. */
267 print_malloc_debug_stats (void)
270 printf ("\nMalloc: %d\nFree: %d\nBalance: %d\n\n",
271 malloc_count, free_count, malloc_count - free_count);
272 for (i = 0; i < ARRAY_SIZE (malloc_debug); i++)
273 if (malloc_debug[i].ptr != NULL)
274 printf ("0x%08ld: %s:%d\n", (long)malloc_debug[i].ptr,
275 malloc_debug[i].file, malloc_debug[i].line);
279 xmalloc_debug (size_t size, const char *source_file, int source_line)
281 void *ptr = xmalloc_real (size);
283 register_ptr (ptr, source_file, source_line);
288 xfree_debug (void *ptr, const char *source_file, int source_line)
290 assert (ptr != NULL);
292 unregister_ptr (ptr);
297 xrealloc_debug (void *ptr, size_t newsize, const char *source_file, int source_line)
299 void *newptr = xrealloc_real (ptr, newsize);
303 register_ptr (newptr, source_file, source_line);
305 else if (newptr != ptr)
307 unregister_ptr (ptr);
308 register_ptr (newptr, source_file, source_line);
314 xstrdup_debug (const char *s, const char *source_file, int source_line)
316 char *copy = xstrdup_real (s);
318 register_ptr (copy, source_file, source_line);
322 #endif /* DEBUG_MALLOC */
324 /* Utility function: like xstrdup(), but also lowercases S. */
327 xstrdup_lower (const char *s)
329 char *copy = xstrdup (s);
336 /* Return a count of how many times CHR occurs in STRING. */
339 count_char (const char *string, char chr)
343 for (p = string; *p; p++)
349 /* Copy the string formed by two pointers (one on the beginning, other
350 on the char after the last char) to a new, malloc-ed location.
353 strdupdelim (const char *beg, const char *end)
355 char *res = (char *)xmalloc (end - beg + 1);
356 memcpy (res, beg, end - beg);
357 res[end - beg] = '\0';
361 /* Parse a string containing comma-separated elements, and return a
362 vector of char pointers with the elements. Spaces following the
363 commas are ignored. */
365 sepstring (const char *s)
379 res = (char **)xrealloc (res, (i + 2) * sizeof (char *));
380 res[i] = strdupdelim (p, s);
383 /* Skip the blanks following the ','. */
391 res = (char **)xrealloc (res, (i + 2) * sizeof (char *));
392 res[i] = strdupdelim (p, s);
397 /* Return pointer to a static char[] buffer in which zero-terminated
398 string-representation of TM (in form hh:mm:ss) is printed.
400 If TM is non-NULL, the current time-in-seconds will be stored
403 (#### This is misleading: one would expect TM would be used instead
404 of the current time in that case. This design was probably
405 influenced by the design time(2), and should be changed at some
406 points. No callers use non-NULL TM anyway.) */
409 time_str (time_t *tm)
411 static char output[15];
413 time_t secs = time (tm);
417 /* In case of error, return the empty string. Maybe we should
418 just abort if this happens? */
422 ptm = localtime (&secs);
423 sprintf (output, "%02d:%02d:%02d", ptm->tm_hour, ptm->tm_min, ptm->tm_sec);
427 /* Like the above, but include the date: YYYY-MM-DD hh:mm:ss. */
430 datetime_str (time_t *tm)
432 static char output[20]; /* "YYYY-MM-DD hh:mm:ss" + \0 */
434 time_t secs = time (tm);
438 /* In case of error, return the empty string. Maybe we should
439 just abort if this happens? */
443 ptm = localtime (&secs);
444 sprintf (output, "%04d-%02d-%02d %02d:%02d:%02d",
445 ptm->tm_year + 1900, ptm->tm_mon + 1, ptm->tm_mday,
446 ptm->tm_hour, ptm->tm_min, ptm->tm_sec);
450 /* The Windows versions of the following two functions are defined in
455 fork_to_background (void)
458 /* Whether we arrange our own version of opt.lfilename here. */
463 opt.lfilename = unique_name (DEFAULT_LOGFILE);
475 /* parent, no error */
476 printf (_("Continuing in background, pid %d.\n"), (int)pid);
478 printf (_("Output will be written to `%s'.\n"), opt.lfilename);
479 exit (0); /* #### should we use _exit()? */
482 /* child: give up the privileges and keep running. */
484 freopen ("/dev/null", "r", stdin);
485 freopen ("/dev/null", "w", stdout);
486 freopen ("/dev/null", "w", stderr);
488 #endif /* not WINDOWS */
490 /* "Touch" FILE, i.e. make its atime and mtime equal to the time
491 specified with TM. */
493 touch (const char *file, time_t tm)
495 #ifdef HAVE_STRUCT_UTIMBUF
496 struct utimbuf times;
497 times.actime = times.modtime = tm;
500 times[0] = times[1] = tm;
503 if (utime (file, ×) == -1)
504 logprintf (LOG_NOTQUIET, "utime(%s): %s\n", file, strerror (errno));
507 /* Checks if FILE is a symbolic link, and removes it if it is. Does
508 nothing under MS-Windows. */
510 remove_link (const char *file)
515 if (lstat (file, &st) == 0 && S_ISLNK (st.st_mode))
517 DEBUGP (("Unlinking %s (symlink).\n", file));
520 logprintf (LOG_VERBOSE, _("Failed to unlink symlink `%s': %s\n"),
521 file, strerror (errno));
526 /* Does FILENAME exist? This is quite a lousy implementation, since
527 it supplies no error codes -- only a yes-or-no answer. Thus it
528 will return that a file does not exist if, e.g., the directory is
529 unreadable. I don't mind it too much currently, though. The
530 proper way should, of course, be to have a third, error state,
531 other than true/false, but that would introduce uncalled-for
532 additional complexity to the callers. */
534 file_exists_p (const char *filename)
537 return access (filename, F_OK) >= 0;
540 return stat (filename, &buf) >= 0;
544 /* Returns 0 if PATH is a directory, 1 otherwise (any kind of file).
545 Returns 0 on error. */
547 file_non_directory_p (const char *path)
550 /* Use lstat() rather than stat() so that symbolic links pointing to
551 directories can be identified correctly. */
552 if (lstat (path, &buf) != 0)
554 return S_ISDIR (buf.st_mode) ? 0 : 1;
557 /* Return a unique filename, given a prefix and count */
559 unique_name_1 (const char *fileprefix, int count)
565 filename = (char *)xmalloc (strlen (fileprefix) + numdigit (count) + 2);
566 sprintf (filename, "%s.%d", fileprefix, count);
569 filename = xstrdup (fileprefix);
571 if (!file_exists_p (filename))
580 /* Return a unique file name, based on PREFIX. */
582 unique_name (const char *prefix)
588 file = unique_name_1 (prefix, count++);
592 /* Create DIRECTORY. If some of the pathname components of DIRECTORY
593 are missing, create them first. In case any mkdir() call fails,
594 return its error status. Returns 0 on successful completion.
596 The behaviour of this function should be identical to the behaviour
597 of `mkdir -p' on systems where mkdir supports the `-p' option. */
599 make_directory (const char *directory)
606 /* Make a copy of dir, to be able to write to it. Otherwise, the
607 function is unsafe if called with a read-only char *argument. */
608 STRDUP_ALLOCA (dir, directory);
610 /* If the first character of dir is '/', skip it (and thus enable
611 creation of absolute-pathname directories. */
612 for (i = (*dir == '/'); 1; ++i)
614 for (; dir[i] && dir[i] != '/'; i++)
619 /* Check whether the directory already exists. Allow creation of
620 of intermediate directories to fail, as the initial path components
621 are not necessarily directories! */
622 if (!file_exists_p (dir))
623 ret = mkdir (dir, 0777);
634 /* Merge BASE with FILE. BASE can be a directory or a file name, FILE
635 should be a file name.
637 file_merge("/foo/bar", "baz") => "/foo/baz"
638 file_merge("/foo/bar/", "baz") => "/foo/bar/baz"
639 file_merge("foo", "bar") => "bar"
641 In other words, it's a simpler and gentler version of uri_merge_1. */
644 file_merge (const char *base, const char *file)
647 const char *cut = (const char *)strrchr (base, '/');
650 return xstrdup (file);
652 result = (char *)xmalloc (cut - base + 1 + strlen (file) + 1);
653 memcpy (result, base, cut - base);
654 result[cut - base] = '/';
655 strcpy (result + (cut - base) + 1, file);
660 static int in_acclist PARAMS ((const char *const *, const char *, int));
662 /* Determine whether a file is acceptable to be followed, according to
663 lists of patterns to accept/reject. */
665 acceptable (const char *s)
669 while (l && s[l] != '/')
676 return (in_acclist ((const char *const *)opt.accepts, s, 1)
677 && !in_acclist ((const char *const *)opt.rejects, s, 1));
679 return in_acclist ((const char *const *)opt.accepts, s, 1);
681 else if (opt.rejects)
682 return !in_acclist ((const char *const *)opt.rejects, s, 1);
686 /* Compare S1 and S2 frontally; S2 must begin with S1. E.g. if S1 is
687 `/something', frontcmp() will return 1 only if S2 begins with
688 `/something'. Otherwise, 0 is returned. */
690 frontcmp (const char *s1, const char *s2)
692 for (; *s1 && *s2 && (*s1 == *s2); ++s1, ++s2);
696 /* Iterate through STRLIST, and return the first element that matches
697 S, through wildcards or front comparison (as appropriate). */
699 proclist (char **strlist, const char *s, enum accd flags)
703 for (x = strlist; *x; x++)
704 if (has_wildcards_p (*x))
706 if (fnmatch (*x, s, FNM_PATHNAME) == 0)
711 char *p = *x + ((flags & ALLABS) && (**x == '/')); /* Remove '/' */
718 /* Returns whether DIRECTORY is acceptable for download, wrt the
719 include/exclude lists.
721 If FLAGS is ALLABS, the leading `/' is ignored in paths; relative
722 and absolute paths may be freely intermixed. */
724 accdir (const char *directory, enum accd flags)
726 /* Remove starting '/'. */
727 if (flags & ALLABS && *directory == '/')
731 if (!proclist (opt.includes, directory, flags))
736 if (proclist (opt.excludes, directory, flags))
742 /* Match the end of STRING against PATTERN. For instance:
744 match_backwards ("abc", "bc") -> 1
745 match_backwards ("abc", "ab") -> 0
746 match_backwards ("abc", "abc") -> 1 */
748 match_tail (const char *string, const char *pattern)
752 for (i = strlen (string), j = strlen (pattern); i >= 0 && j >= 0; i--, j--)
753 if (string[i] != pattern[j])
755 /* If the pattern was exhausted, the match was succesful. */
762 /* Checks whether string S matches each element of ACCEPTS. A list
763 element are matched either with fnmatch() or match_tail(),
764 according to whether the element contains wildcards or not.
766 If the BACKWARD is 0, don't do backward comparison -- just compare
769 in_acclist (const char *const *accepts, const char *s, int backward)
771 for (; *accepts; accepts++)
773 if (has_wildcards_p (*accepts))
775 /* fnmatch returns 0 if the pattern *does* match the
777 if (fnmatch (*accepts, s, 0) == 0)
784 if (match_tail (s, *accepts))
789 if (!strcmp (s, *accepts))
797 /* Return the location of STR's suffix (file extension). Examples:
798 suffix ("foo.bar") -> "bar"
799 suffix ("foo.bar.baz") -> "baz"
800 suffix ("/foo/bar") -> NULL
801 suffix ("/foo.bar/baz") -> NULL */
803 suffix (const char *str)
807 for (i = strlen (str); i && str[i] != '/' && str[i] != '.'; i--)
811 return (char *)str + i;
816 /* Return non-zero if FNAME ends with a typical HTML suffix. The
817 following (case-insensitive) suffixes are presumed to be HTML files:
821 ?html (`?' matches one character)
823 #### CAVEAT. This is not necessarily a good indication that FNAME
824 refers to a file that contains HTML! */
826 has_html_suffix_p (const char *fname)
830 if ((suf = suffix (fname)) == NULL)
832 if (!strcasecmp (suf, "html"))
834 if (!strcasecmp (suf, "htm"))
836 if (suf[0] && !strcasecmp (suf + 1, "html"))
841 /* Read a line from FP and return the pointer to freshly allocated
842 storage. The stoarage space is obtained through malloc() and
843 should be freed with free() when it is no longer needed.
845 The length of the line is not limited, except by available memory.
846 The newline character at the end of line is retained. The line is
847 terminated with a zero character.
849 After end-of-file is encountered without anything being read, NULL
850 is returned. NULL is also returned on error. To distinguish
851 between these two cases, use the stdio function ferror(). */
854 read_whole_line (FILE *fp)
858 char *line = (char *)xmalloc (bufsize);
860 while (fgets (line + length, bufsize - length, fp))
862 length += strlen (line + length);
864 /* Possible for example when reading from a binary file where
865 a line begins with \0. */
868 if (line[length - 1] == '\n')
871 /* fgets() guarantees to read the whole line, or to use up the
872 space we've given it. We can double the buffer
875 line = xrealloc (line, bufsize);
877 if (length == 0 || ferror (fp))
882 if (length + 1 < bufsize)
883 /* Relieve the memory from our exponential greediness. We say
884 `length + 1' because the terminating \0 is not included in
885 LENGTH. We don't need to zero-terminate the string ourselves,
886 though, because fgets() does that. */
887 line = xrealloc (line, length + 1);
891 /* Read FILE into memory. A pointer to `struct file_memory' are
892 returned; use struct element `content' to access file contents, and
893 the element `length' to know the file length. `content' is *not*
894 zero-terminated, and you should *not* read or write beyond the [0,
895 length) range of characters.
897 After you are done with the file contents, call read_file_free to
900 Depending on the operating system and the type of file that is
901 being read, read_file() either mmap's the file into memory, or
902 reads the file into the core using read().
904 If file is named "-", fileno(stdin) is used for reading instead.
905 If you want to read from a real file named "-", use "./-" instead. */
908 read_file (const char *file)
911 struct file_memory *fm;
913 int inhibit_close = 0;
915 /* Some magic in the finest tradition of Perl and its kin: if FILE
916 is "-", just use stdin. */
921 /* Note that we don't inhibit mmap() in this case. If stdin is
922 redirected from a regular file, mmap() will still work. */
925 fd = open (file, O_RDONLY);
928 fm = xmalloc (sizeof (struct file_memory));
933 if (fstat (fd, &buf) < 0)
935 fm->length = buf.st_size;
936 /* NOTE: As far as I know, the callers of this function never
937 modify the file text. Relying on this would enable us to
938 specify PROT_READ and MAP_SHARED for a marginal gain in
939 efficiency, but at some cost to generality. */
940 fm->content = mmap (NULL, fm->length, PROT_READ | PROT_WRITE,
942 if (fm->content == (char *)MAP_FAILED)
952 /* The most common reason why mmap() fails is that FD does not point
953 to a plain file. However, it's also possible that mmap() doesn't
954 work for a particular type of file. Therefore, whenever mmap()
955 fails, we just fall back to the regular method. */
956 #endif /* HAVE_MMAP */
959 size = 512; /* number of bytes fm->contents can
960 hold at any given time. */
961 fm->content = xmalloc (size);
965 if (fm->length > size / 2)
967 /* #### I'm not sure whether the whole exponential-growth
968 thing makes sense with kernel read. On Linux at least,
969 read() refuses to read more than 4K from a file at a
970 single chunk anyway. But other Unixes might optimize it
971 better, and it doesn't *hurt* anything, so I'm leaving
974 /* Normally, we grow SIZE exponentially to make the number
975 of calls to read() and realloc() logarithmic in relation
976 to file size. However, read() can read an amount of data
977 smaller than requested, and it would be unreasonably to
978 double SIZE every time *something* was read. Therefore,
979 we double SIZE only when the length exceeds half of the
980 entire allocated size. */
982 fm->content = xrealloc (fm->content, size);
984 nread = read (fd, fm->content + fm->length, size - fm->length);
986 /* Successful read. */
997 if (size > fm->length && fm->length != 0)
998 /* Due to exponential growth of fm->content, the allocated region
999 might be much larger than what is actually needed. */
1000 fm->content = xrealloc (fm->content, fm->length);
1007 xfree (fm->content);
1012 /* Release the resources held by FM. Specifically, this calls
1013 munmap() or xfree() on fm->content, depending whether mmap or
1014 malloc/read were used to read in the file. It also frees the
1015 memory needed to hold the FM structure itself. */
1018 read_file_free (struct file_memory *fm)
1023 munmap (fm->content, fm->length);
1028 xfree (fm->content);
1033 /* Free the pointers in a NULL-terminated vector of pointers, then
1034 free the pointer itself. */
1036 free_vec (char **vec)
1047 /* Append vector V2 to vector V1. The function frees V2 and
1048 reallocates V1 (thus you may not use the contents of neither
1049 pointer after the call). If V1 is NULL, V2 is returned. */
1051 merge_vecs (char **v1, char **v2)
1061 /* To avoid j == 0 */
1066 for (i = 0; v1[i]; i++);
1068 for (j = 0; v2[j]; j++);
1069 /* Reallocate v1. */
1070 v1 = (char **)xrealloc (v1, (i + j + 1) * sizeof (char **));
1071 memcpy (v1 + i, v2, (j + 1) * sizeof (char *));
1076 /* A set of simple-minded routines to store strings in a linked list.
1077 This used to also be used for searching, but now we have hash
1080 /* It's a shame that these simple things like linked lists and hash
1081 tables (see hash.c) need to be implemented over and over again. It
1082 would be nice to be able to use the routines from glib -- see
1083 www.gtk.org for details. However, that would make Wget depend on
1084 glib, and I want to avoid dependencies to external libraries for
1085 reasons of convenience and portability (I suspect Wget is more
1086 portable than anything ever written for Gnome). */
1088 /* Append an element to the list. If the list has a huge number of
1089 elements, this can get slow because it has to find the list's
1090 ending. If you think you have to call slist_append in a loop,
1091 think about calling slist_prepend() followed by slist_nreverse(). */
1094 slist_append (slist *l, const char *s)
1096 slist *newel = (slist *)xmalloc (sizeof (slist));
1099 newel->string = xstrdup (s);
1104 /* Find the last element. */
1111 /* Prepend S to the list. Unlike slist_append(), this is O(1). */
1114 slist_prepend (slist *l, const char *s)
1116 slist *newel = (slist *)xmalloc (sizeof (slist));
1117 newel->string = xstrdup (s);
1122 /* Destructively reverse L. */
1125 slist_nreverse (slist *l)
1130 slist *next = l->next;
1138 /* Is there a specific entry in the list? */
1140 slist_contains (slist *l, const char *s)
1142 for (; l; l = l->next)
1143 if (!strcmp (l->string, s))
1148 /* Free the whole slist. */
1150 slist_free (slist *l)
1161 /* Sometimes it's useful to create "sets" of strings, i.e. special
1162 hash tables where you want to store strings as keys and merely
1163 query for their existence. Here is a set of utility routines that
1164 makes that transparent. */
1167 string_set_add (struct hash_table *ht, const char *s)
1169 /* First check whether the set element already exists. If it does,
1170 do nothing so that we don't have to free() the old element and
1171 then strdup() a new one. */
1172 if (hash_table_contains (ht, s))
1175 /* We use "1" as value. It provides us a useful and clear arbitrary
1176 value, and it consumes no memory -- the pointers to the same
1177 string "1" will be shared by all the key-value pairs in all `set'
1179 hash_table_put (ht, xstrdup (s), "1");
1182 /* Synonym for hash_table_contains... */
1185 string_set_contains (struct hash_table *ht, const char *s)
1187 return hash_table_contains (ht, s);
1191 string_set_free_mapper (void *key, void *value_ignored, void *arg_ignored)
1198 string_set_free (struct hash_table *ht)
1200 hash_table_map (ht, string_set_free_mapper, NULL);
1201 hash_table_destroy (ht);
1205 free_keys_and_values_mapper (void *key, void *value, void *arg_ignored)
1212 /* Another utility function: call free() on all keys and values of HT. */
1215 free_keys_and_values (struct hash_table *ht)
1217 hash_table_map (ht, free_keys_and_values_mapper, NULL);
1221 /* Engine for legible and legible_very_long; this function works on
1225 legible_1 (const char *repr)
1227 static char outbuf[128];
1232 /* Reset the pointers. */
1235 /* If the number is negative, shift the pointers. */
1241 /* How many digits before the first separator? */
1242 mod = strlen (inptr) % 3;
1244 for (i = 0; i < mod; i++)
1245 *outptr++ = inptr[i];
1246 /* Now insert the rest of them, putting separator before every
1248 for (i1 = i, i = 0; inptr[i1]; i++, i1++)
1250 if (i % 3 == 0 && i1 != 0)
1252 *outptr++ = inptr[i1];
1254 /* Zero-terminate the string. */
1259 /* Legible -- return a static pointer to the legibly printed long. */
1264 /* Print the number into the buffer. */
1265 number_to_string (inbuf, l);
1266 return legible_1 (inbuf);
1269 /* Write a string representation of NUMBER into the provided buffer.
1270 We cannot use sprintf() because we cannot be sure whether the
1271 platform supports printing of what we chose for VERY_LONG_TYPE.
1273 Example: Gcc supports `long long' under many platforms, but on many
1274 of those the native libc knows nothing of it and therefore cannot
1277 How long BUFFER needs to be depends on the platform and the content
1278 of NUMBER. For 64-bit VERY_LONG_TYPE (the most common case), 24
1279 bytes are sufficient. Using more might be a good idea.
1281 This function does not go through the hoops that long_to_string
1282 goes to because it doesn't aspire to be fast. (It's called perhaps
1283 once in a Wget run.) */
1286 very_long_to_string (char *buffer, VERY_LONG_TYPE number)
1291 /* Print the number backwards... */
1294 buffer[i++] = '0' + number % 10;
1299 /* ...and reverse the order of the digits. */
1300 for (j = 0; j < i / 2; j++)
1303 buffer[j] = buffer[i - 1 - j];
1304 buffer[i - 1 - j] = c;
1309 /* The same as legible(), but works on VERY_LONG_TYPE. See sysdep.h. */
1311 legible_very_long (VERY_LONG_TYPE l)
1314 /* Print the number into the buffer. */
1315 very_long_to_string (inbuf, l);
1316 return legible_1 (inbuf);
1319 /* Count the digits in a (long) integer. */
1321 numdigit (long number)
1329 while ((number /= 10) > 0)
1334 #define ONE_DIGIT(figure) *p++ = n / (figure) + '0'
1335 #define ONE_DIGIT_ADVANCE(figure) (ONE_DIGIT (figure), n %= (figure))
1337 #define DIGITS_1(figure) ONE_DIGIT (figure)
1338 #define DIGITS_2(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_1 ((figure) / 10)
1339 #define DIGITS_3(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_2 ((figure) / 10)
1340 #define DIGITS_4(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_3 ((figure) / 10)
1341 #define DIGITS_5(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_4 ((figure) / 10)
1342 #define DIGITS_6(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_5 ((figure) / 10)
1343 #define DIGITS_7(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_6 ((figure) / 10)
1344 #define DIGITS_8(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_7 ((figure) / 10)
1345 #define DIGITS_9(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_8 ((figure) / 10)
1346 #define DIGITS_10(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_9 ((figure) / 10)
1348 /* DIGITS_<11-20> are only used on machines with 64-bit longs. */
1350 #define DIGITS_11(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_10 ((figure) / 10)
1351 #define DIGITS_12(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_11 ((figure) / 10)
1352 #define DIGITS_13(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_12 ((figure) / 10)
1353 #define DIGITS_14(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_13 ((figure) / 10)
1354 #define DIGITS_15(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_14 ((figure) / 10)
1355 #define DIGITS_16(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_15 ((figure) / 10)
1356 #define DIGITS_17(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_16 ((figure) / 10)
1357 #define DIGITS_18(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_17 ((figure) / 10)
1358 #define DIGITS_19(figure) ONE_DIGIT_ADVANCE (figure); DIGITS_18 ((figure) / 10)
1360 /* Print NUMBER to BUFFER in base 10. This should be completely
1361 equivalent to `sprintf(buffer, "%ld", number)', only much faster.
1363 The speedup may make a difference in programs that frequently
1364 convert numbers to strings. Some implementations of sprintf,
1365 particularly the one in GNU libc, have been known to be extremely
1366 slow compared to this function.
1368 Return the pointer to the location where the terminating zero was
1369 printed. (Equivalent to calling buffer+strlen(buffer) after the
1372 BUFFER should be big enough to accept as many bytes as you expect
1373 the number to take up. On machines with 64-bit longs the maximum
1374 needed size is 24 bytes. That includes the digits needed for the
1375 largest 64-bit number, the `-' sign in case it's negative, and the
1376 terminating '\0'. */
1379 number_to_string (char *buffer, long number)
1384 #if (SIZEOF_LONG != 4) && (SIZEOF_LONG != 8)
1385 /* We are running in a strange or misconfigured environment. Let
1386 sprintf cope with it. */
1387 sprintf (buffer, "%ld", n);
1388 p += strlen (buffer);
1389 #else /* (SIZEOF_LONG == 4) || (SIZEOF_LONG == 8) */
1397 if (n < 10) { DIGITS_1 (1); }
1398 else if (n < 100) { DIGITS_2 (10); }
1399 else if (n < 1000) { DIGITS_3 (100); }
1400 else if (n < 10000) { DIGITS_4 (1000); }
1401 else if (n < 100000) { DIGITS_5 (10000); }
1402 else if (n < 1000000) { DIGITS_6 (100000); }
1403 else if (n < 10000000) { DIGITS_7 (1000000); }
1404 else if (n < 100000000) { DIGITS_8 (10000000); }
1405 else if (n < 1000000000) { DIGITS_9 (100000000); }
1406 #if SIZEOF_LONG == 4
1407 /* ``if (1)'' serves only to preserve editor indentation. */
1408 else if (1) { DIGITS_10 (1000000000); }
1409 #else /* SIZEOF_LONG != 4 */
1410 else if (n < 10000000000L) { DIGITS_10 (1000000000L); }
1411 else if (n < 100000000000L) { DIGITS_11 (10000000000L); }
1412 else if (n < 1000000000000L) { DIGITS_12 (100000000000L); }
1413 else if (n < 10000000000000L) { DIGITS_13 (1000000000000L); }
1414 else if (n < 100000000000000L) { DIGITS_14 (10000000000000L); }
1415 else if (n < 1000000000000000L) { DIGITS_15 (100000000000000L); }
1416 else if (n < 10000000000000000L) { DIGITS_16 (1000000000000000L); }
1417 else if (n < 100000000000000000L) { DIGITS_17 (10000000000000000L); }
1418 else if (n < 1000000000000000000L) { DIGITS_18 (100000000000000000L); }
1419 else { DIGITS_19 (1000000000000000000L); }
1420 #endif /* SIZEOF_LONG != 4 */
1423 #endif /* (SIZEOF_LONG == 4) || (SIZEOF_LONG == 8) */
1429 #undef ONE_DIGIT_ADVANCE
1451 /* Support for timers. */
1453 #undef TIMER_WINDOWS
1454 #undef TIMER_GETTIMEOFDAY
1457 /* Depending on the OS and availability of gettimeofday(), one and
1458 only one of the above constants will be defined. Virtually all
1459 modern Unix systems will define TIMER_GETTIMEOFDAY; Windows will
1460 use TIMER_WINDOWS. TIMER_TIME is a catch-all method for
1461 non-Windows systems without gettimeofday.
1463 #### Perhaps we should also support ftime(), which exists on old
1464 BSD 4.2-influenced systems? (It also existed under MS DOS Borland
1465 C, if memory serves me.) */
1468 # define TIMER_WINDOWS
1469 #else /* not WINDOWS */
1470 # ifdef HAVE_GETTIMEOFDAY
1471 # define TIMER_GETTIMEOFDAY
1475 #endif /* not WINDOWS */
1478 #ifdef TIMER_GETTIMEOFDAY
1487 #ifdef TIMER_WINDOWS
1488 ULARGE_INTEGER wintime;
1492 /* Allocate a timer. It is not legal to do anything with a freshly
1493 allocated timer, except call wtimer_reset() or wtimer_delete(). */
1496 wtimer_allocate (void)
1498 struct wget_timer *wt =
1499 (struct wget_timer *)xmalloc (sizeof (struct wget_timer));
1503 /* Allocate a new timer and reset it. Return the new timer. */
1508 struct wget_timer *wt = wtimer_allocate ();
1513 /* Free the resources associated with the timer. Its further use is
1517 wtimer_delete (struct wget_timer *wt)
1522 /* Reset timer WT. This establishes the starting point from which
1523 wtimer_elapsed() will return the number of elapsed
1524 milliseconds. It is allowed to reset a previously used timer. */
1527 wtimer_reset (struct wget_timer *wt)
1529 #ifdef TIMER_GETTIMEOFDAY
1531 gettimeofday (&t, NULL);
1532 wt->secs = t.tv_sec;
1533 wt->usecs = t.tv_usec;
1537 wt->secs = time (NULL);
1540 #ifdef TIMER_WINDOWS
1543 GetSystemTime (&st);
1544 SystemTimeToFileTime (&st, &ft);
1545 wt->wintime.HighPart = ft.dwHighDateTime;
1546 wt->wintime.LowPart = ft.dwLowDateTime;
1550 /* Return the number of milliseconds elapsed since the timer was last
1551 reset. It is allowed to call this function more than once to get
1552 increasingly higher elapsed values. */
1555 wtimer_elapsed (struct wget_timer *wt)
1557 #ifdef TIMER_GETTIMEOFDAY
1559 gettimeofday (&t, NULL);
1560 return (t.tv_sec - wt->secs) * 1000 + (t.tv_usec - wt->usecs) / 1000;
1564 time_t now = time (NULL);
1565 return 1000 * (now - wt->secs);
1572 GetSystemTime (&st);
1573 SystemTimeToFileTime (&st, &ft);
1574 uli.HighPart = ft.dwHighDateTime;
1575 uli.LowPart = ft.dwLowDateTime;
1576 return (long)((uli.QuadPart - wt->wintime.QuadPart) / 10000);
1580 /* Return the assessed granularity of the timer implementation. This
1581 is important for certain code that tries to deal with "zero" time
1585 wtimer_granularity (void)
1587 #ifdef TIMER_GETTIMEOFDAY
1588 /* Granularity of gettimeofday is hugely architecture-dependent.
1589 However, it appears that on modern machines it is better than
1595 /* This is clear. */
1599 #ifdef TIMER_WINDOWS
1605 /* This should probably be at a better place, but it doesn't really
1606 fit into html-parse.c. */
1608 /* The function returns the pointer to the malloc-ed quoted version of
1609 string s. It will recognize and quote numeric and special graphic
1610 entities, as per RFC1866:
1618 No other entities are recognized or replaced. */
1620 html_quote_string (const char *s)
1626 /* Pass through the string, and count the new size. */
1627 for (i = 0; *s; s++, i++)
1630 i += 4; /* `amp;' */
1631 else if (*s == '<' || *s == '>')
1632 i += 3; /* `lt;' and `gt;' */
1633 else if (*s == '\"')
1634 i += 5; /* `quot;' */
1638 res = (char *)xmalloc (i + 1);
1640 for (p = res; *s; s++)
1653 *p++ = (*s == '<' ? 'l' : 'g');
1680 /* Determine the width of the terminal we're running on. If that's
1681 not possible, return 0. */
1684 determine_screen_width (void)
1686 /* If there's a way to get the terminal size using POSIX
1687 tcgetattr(), somebody please tell me. */
1690 #else /* TIOCGWINSZ */
1694 if (opt.lfilename != NULL)
1697 fd = fileno (stderr);
1698 if (ioctl (fd, TIOCGWINSZ, &wsz) < 0)
1699 return 0; /* most likely ENOTTY */
1702 #endif /* TIOCGWINSZ */
1705 /* Return a random number between 0 and MAX-1, inclusive.
1707 If MAX is greater than the value of RAND_MAX+1 on the system, the
1708 returned value will be in the range [0, RAND_MAX]. This may be
1709 fixed in a future release.
1711 The random number generator is seeded automatically the first time
1714 This uses rand() for portability. It has been suggested that
1715 random() offers better randomness, but this is not required for
1716 Wget, so I chose to go for simplicity and use rand
1720 random_number (int max)
1728 srand (time (NULL));
1733 /* On systems that don't define RAND_MAX, assume it to be 2**15 - 1,
1734 and enforce that assumption by masking other bits. */
1736 # define RAND_MAX 32767
1740 /* This is equivalent to rand() % max, but uses the high-order bits
1741 for better randomness on architecture where rand() is implemented
1742 using a simple congruential generator. */
1744 bounded = (double)max * rnd / (RAND_MAX + 1.0);
1745 return (int)bounded;
1749 /* A debugging function for checking whether an MD5 library works. */
1751 #include "gen-md5.h"
1754 debug_test_md5 (char *buf)
1756 unsigned char raw[16];
1757 static char res[33];
1761 ALLOCA_MD5_CONTEXT (ctx);
1764 gen_md5_update ((unsigned char *)buf, strlen (buf), ctx);
1765 gen_md5_finish (ctx, raw);
1772 *p2++ = XDIGIT_TO_xchar (*p1 >> 4);
1773 *p2++ = XDIGIT_TO_xchar (*p1 & 0xf);
1782 /* Implementation of run_with_timeout, a generic timeout handler for
1783 systems with Unix-like signal handling. */
1784 #ifdef HAVE_SIGSETJMP
1785 #define SETJMP(env) sigsetjmp (env, 1)
1787 static sigjmp_buf run_with_timeout_env;
1790 abort_run_with_timeout (int sig)
1792 assert (sig == SIGALRM);
1793 siglongjmp (run_with_timeout_env, -1);
1795 #else /* not HAVE_SIGSETJMP */
1796 #define SETJMP(env) setjmp (env)
1798 static jmp_buf run_with_timeout_env;
1801 abort_run_with_timeout (int sig)
1803 assert (sig == SIGALRM);
1804 /* We don't have siglongjmp to preserve the set of blocked signals;
1805 if we longjumped out of the handler at this point, SIGALRM would
1806 remain blocked. We must unblock it manually. */
1807 int mask = siggetmask ();
1808 mask &= ~sigmask(SIGALRM);
1811 /* Now it's safe to longjump. */
1812 longjmp (run_with_timeout_env, -1);
1814 #endif /* not HAVE_SIGSETJMP */
1817 run_with_timeout (long timeout, void (*fun) (void *), void *arg)
1819 #ifndef USE_SIGNAL_TIMEOUT
1831 signal (SIGALRM, abort_run_with_timeout);
1832 if (SETJMP (run_with_timeout_env) != 0)
1834 /* Longjumped out of FUN with a timeout. */
1835 signal (SIGALRM, SIG_DFL);
1841 /* Preserve errno in case alarm() or signal() modifies it. */
1842 saved_errno = errno;
1844 signal (SIGALRM, SIG_DFL);
1845 errno = saved_errno;