#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
+#ifdef HAVE_MMAP
+# include <sys/mman.h>
+#endif
#ifdef HAVE_PWD_H
# include <pwd.h>
#endif
#ifdef NeXT
# include <libc.h> /* for access() */
#endif
+#include <fcntl.h>
#include <assert.h>
#include "wget.h"
#include "utils.h"
#include "fnmatch.h"
+#include "hash.h"
#ifndef errno
extern int errno;
#endif
+/* This section implements several wrappers around the basic
+ allocation routines. This is done for two reasons: first, so that
+ the callers of these functions need not consistently check for
+ errors. If there is not enough virtual memory for running Wget,
+ something is seriously wrong, and Wget exits with an appropriate
+ error message.
+
+ The second reason why these are useful is that, if DEBUG_MALLOC is
+ defined, they also provide a handy (if crude) malloc debugging
+ interface that checks memory leaks. */
/* Croak the fatal memory error and bail out with non-zero exit
status. */
static void
-memfatal (const char *s)
+memfatal (const char *what)
{
/* HACK: expose save_log_p from log.c, so we can turn it off in
order to prevent saving the log. Saving the log is dangerous
because logprintf() and logputs() can call malloc(), so this
could infloop. When logging is turned off, infloop can no longer
- happen. */
+ happen.
+
+ #### This is no longer really necessary because the new routines
+ in log.c cons only if the line exceeds eighty characters. But
+ this can come at the end of a line, so it's OK to be careful.
+
+ On a more serious note, it would be good to have a
+ log_forced_shutdown() routine that exposes this cleanly. */
extern int save_log_p;
save_log_p = 0;
- logprintf (LOG_ALWAYS, _("%s: %s: Not enough memory.\n"), exec_name, s);
+ logprintf (LOG_ALWAYS, _("%s: %s: Not enough memory.\n"), exec_name, what);
exit (1);
}
-/* xmalloc, xrealloc and xstrdup exit the program if there is not
- enough memory. xstrdup also implements strdup on systems that do
- not have it. */
+/* These functions end with _real because they need to be
+ distinguished from the debugging functions, and from the macros.
+ Explanation follows:
+
+ If memory debugging is not turned on, wget.h defines these:
+
+ #define xmalloc xmalloc_real
+ #define xfree xfree_real
+ #define xrealloc xrealloc_real
+ #define xstrdup xstrdup_real
+
+ In case of memory debugging, the definitions are a bit more
+ complex, because we want to provide more information, *and* we want
+ to call the debugging code. (The former is the reason why xmalloc
+ and friends need to be macros in the first place.) Then it looks
+ like this:
+
+ #define xmalloc(a) xmalloc_debug (a, __FILE__, __LINE__)
+ #define xfree(a) xfree_debug (a, __FILE__, __LINE__)
+ #define xrealloc(a, b) xrealloc_debug (a, b, __FILE__, __LINE__)
+ #define xstrdup(a) xstrdup_debug (a, __FILE__, __LINE__)
+
+ Each of the *_debug function does its magic and calls the real one. */
+
void *
-xmalloc (size_t size)
+xmalloc_real (size_t size)
{
- void *res;
-
- res = malloc (size);
- if (!res)
+ void *ptr = malloc (size);
+ if (!ptr)
memfatal ("malloc");
- return res;
+ return ptr;
+}
+
+void
+xfree_real (void *ptr)
+{
+ free (ptr);
}
void *
-xrealloc (void *obj, size_t size)
+xrealloc_real (void *ptr, size_t newsize)
{
- void *res;
+ void *newptr;
/* Not all Un*xes have the feature of realloc() that calling it with
a NULL-pointer is the same as malloc(), but it is easy to
simulate. */
- if (obj)
- res = realloc (obj, size);
+ if (ptr)
+ newptr = realloc (ptr, newsize);
else
- res = malloc (size);
- if (!res)
+ newptr = malloc (newsize);
+ if (!newptr)
memfatal ("realloc");
- return res;
+ return newptr;
}
char *
-xstrdup (const char *s)
+xstrdup_real (const char *s)
{
+ char *copy;
+
#ifndef HAVE_STRDUP
int l = strlen (s);
- char *s1 = malloc (l + 1);
- if (!s1)
+ copy = malloc (l + 1);
+ if (!copy)
memfatal ("strdup");
- memcpy (s1, s, l + 1);
- return s1;
+ memcpy (copy, s, l + 1);
#else /* HAVE_STRDUP */
- char *s1 = strdup (s);
- if (!s1)
+ copy = strdup (s);
+ if (!copy)
memfatal ("strdup");
- return s1;
#endif /* HAVE_STRDUP */
+
+ return copy;
}
+
+#ifdef DEBUG_MALLOC
+
+/* Crude home-grown routines for debugging some malloc-related
+ problems. Featured:
+
+ * Counting the number of malloc and free invocations, and reporting
+ the "balance", i.e. how many times more malloc was called than it
+ was the case with free.
+
+ * Making malloc store its entry into a simple array and free remove
+ stuff from that array. At the end, print the pointers which have
+ not been freed, along with the source file and the line number.
+ This also has the side-effect of detecting freeing memory that
+ was never allocated.
+
+ Note that this kind of memory leak checking strongly depends on
+ every malloc() being followed by a free(), even if the program is
+ about to finish. Wget is careful to free the data structure it
+ allocated in init.c. */
+
+static int malloc_count, free_count;
+
+static struct {
+ char *ptr;
+ const char *file;
+ int line;
+} malloc_debug[100000];
+
+/* Both register_ptr and unregister_ptr take O(n) operations to run,
+ which can be a real problem. It would be nice to use a hash table
+ for malloc_debug, but the functions in hash.c are not suitable
+ because they can call malloc() themselves. Maybe it would work if
+ the hash table were preallocated to a huge size, and if we set the
+ rehash threshold to 1.0. */
+
+/* Register PTR in malloc_debug. Abort if this is not possible
+ (presumably due to the number of current allocations exceeding the
+ size of malloc_debug.) */
+
+static void
+register_ptr (void *ptr, const char *file, int line)
+{
+ int i;
+ for (i = 0; i < ARRAY_SIZE (malloc_debug); i++)
+ if (malloc_debug[i].ptr == NULL)
+ {
+ malloc_debug[i].ptr = ptr;
+ malloc_debug[i].file = file;
+ malloc_debug[i].line = line;
+ return;
+ }
+ abort ();
+}
+
+/* Unregister PTR from malloc_debug. Abort if PTR is not present in
+ malloc_debug. (This catches calling free() with a bogus pointer.) */
+
+static void
+unregister_ptr (void *ptr)
+{
+ int i;
+ for (i = 0; i < ARRAY_SIZE (malloc_debug); i++)
+ if (malloc_debug[i].ptr == ptr)
+ {
+ malloc_debug[i].ptr = NULL;
+ return;
+ }
+ abort ();
+}
+
+/* Print the malloc debug stats that can be gathered from the above
+ information. Currently this is the count of mallocs, frees, the
+ difference between the two, and the dump of the contents of
+ malloc_debug. The last part are the memory leaks. */
+
+void
+print_malloc_debug_stats (void)
+{
+ int i;
+ printf ("\nMalloc: %d\nFree: %d\nBalance: %d\n\n",
+ malloc_count, free_count, malloc_count - free_count);
+ for (i = 0; i < ARRAY_SIZE (malloc_debug); i++)
+ if (malloc_debug[i].ptr != NULL)
+ printf ("0x%08ld: %s:%d\n", (long)malloc_debug[i].ptr,
+ malloc_debug[i].file, malloc_debug[i].line);
+}
+
+void *
+xmalloc_debug (size_t size, const char *source_file, int source_line)
+{
+ void *ptr = xmalloc_real (size);
+ ++malloc_count;
+ register_ptr (ptr, source_file, source_line);
+ return ptr;
+}
+
+void
+xfree_debug (void *ptr, const char *source_file, int source_line)
+{
+ assert (ptr != NULL);
+ ++free_count;
+ unregister_ptr (ptr);
+ xfree_real (ptr);
+}
+
+void *
+xrealloc_debug (void *ptr, size_t newsize, const char *source_file, int source_line)
+{
+ void *newptr = xrealloc_real (ptr, newsize);
+ if (!ptr)
+ {
+ ++malloc_count;
+ register_ptr (newptr, source_file, source_line);
+ }
+ else
+ {
+ unregister_ptr (ptr);
+ register_ptr (newptr, source_file, source_line);
+ }
+ return newptr;
+}
+
+char *
+xstrdup_debug (const char *s, const char *source_file, int source_line)
+{
+ char *copy = xstrdup_real (s);
+ ++malloc_count;
+ register_ptr (copy, source_file, source_line);
+ return copy;
+}
+
+#endif /* DEBUG_MALLOC */
\f
/* Copy the string formed by two pointers (one on the beginning, other
on the char after the last char) to a new, malloc-ed location.
return filename;
else
{
- free (filename);
+ xfree (filename);
return NULL;
}
}
}
if (length == 0 || ferror (fp))
{
- free (line);
+ xfree (line);
return NULL;
}
if (length + 1 < bufsize)
line = xrealloc (line, length + 1);
return line;
}
+\f
+/* Read FILE into memory. A pointer to `struct file_memory' are
+ returned; use struct element `content' to access file contents, and
+ the element `length' to know the file length. `content' is *not*
+ zero-terminated, and you should *not* read or write beyond the [0,
+ length) range of characters.
-/* Load file pointed to by FP to memory and return the malloc-ed
- buffer with the contents. *NREAD will contain the number of read
- bytes. The file is loaded in chunks, allocated exponentially,
- starting with FILE_BUFFER_SIZE bytes. */
-void
-load_file (FILE *fp, char **buf, long *nread)
-{
- long bufsize;
+ After you are done with the file contents, call read_file_free to
+ release the memory.
+
+ Depending on the operating system and the type of file that is
+ being read, read_file() either mmap's the file into memory, or
+ reads the file into the core using read().
+
+ If file is named "-", fileno(stdin) is used for reading instead.
+ If you want to read from a real file named "-", use "./-" instead. */
- bufsize = 512;
- *nread = 0;
- *buf = NULL;
- while (!feof (fp) && !ferror (fp))
+struct file_memory *
+read_file (const char *file)
+{
+ int fd;
+ struct file_memory *fm;
+ long size;
+ int inhibit_close = 0;
+
+ /* Some magic in the finest tradition of Perl and its kin: if FILE
+ is "-", just use stdin. */
+ if (HYPHENP (file))
{
- *buf = (char *)xrealloc (*buf, bufsize + *nread);
- *nread += fread (*buf + *nread, sizeof (char), bufsize, fp);
- bufsize <<= 1;
+ fd = fileno (stdin);
+ inhibit_close = 1;
+ /* Note that we don't inhibit mmap() in this case. If stdin is
+ redirected from a regular file, mmap() will still work. */
}
- /* #### No indication of encountered error?? */
+ else
+ fd = open (file, O_RDONLY);
+ if (fd < 0)
+ return NULL;
+ fm = xmalloc (sizeof (struct file_memory));
+
+#ifdef HAVE_MMAP
+ {
+ struct stat buf;
+ if (fstat (fd, &buf) < 0)
+ goto mmap_lose;
+ fm->length = buf.st_size;
+ /* NOTE: As far as I know, the callers of this function never
+ modify the file text. Relying on this would enable us to
+ specify PROT_READ and MAP_SHARED for a marginal gain in
+ efficiency, but at some cost to generality. */
+ fm->content = mmap (NULL, fm->length, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE, fd, 0);
+ if (fm->content == MAP_FAILED)
+ goto mmap_lose;
+ if (!inhibit_close)
+ close (fd);
+
+ fm->mmap_p = 1;
+ return fm;
+ }
+
+ mmap_lose:
+ /* The most common reason why mmap() fails is that FD does not point
+ to a plain file. However, it's also possible that mmap() doesn't
+ work for a particular type of file. Therefore, whenever mmap()
+ fails, we just fall back to the regular method. */
+#endif /* HAVE_MMAP */
+
+ fm->length = 0;
+ size = 512; /* number of bytes fm->contents can
+ hold at any given time. */
+ fm->content = xmalloc (size);
+ while (1)
+ {
+ long nread;
+ if (fm->length > size / 2)
+ {
+ /* #### I'm not sure whether the whole exponential-growth
+ thing makes sense with kernel read. On Linux at least,
+ read() refuses to read more than 4K from a file at a
+ single chunk anyway. But other Unixes might optimize it
+ better, and it doesn't *hurt* anything, so I'm leaving
+ it. */
+
+ /* Normally, we grow SIZE exponentially to make the number
+ of calls to read() and realloc() logarithmic in relation
+ to file size. However, read() can read an amount of data
+ smaller than requested, and it would be unreasonably to
+ double SIZE every time *something* was read. Therefore,
+ we double SIZE only when the length exceeds half of the
+ entire allocated size. */
+ size <<= 1;
+ fm->content = xrealloc (fm->content, size);
+ }
+ nread = read (fd, fm->content + fm->length, size - fm->length);
+ if (nread > 0)
+ /* Successful read. */
+ fm->length += nread;
+ else if (nread < 0)
+ /* Error. */
+ goto lose;
+ else
+ /* EOF */
+ break;
+ }
+ if (!inhibit_close)
+ close (fd);
+ if (size > fm->length && fm->length != 0)
+ /* Due to exponential growth of fm->content, the allocated region
+ might be much larger than what is actually needed. */
+ fm->content = xrealloc (fm->content, fm->length);
+ fm->mmap_p = 0;
+ return fm;
+
+ lose:
+ if (!inhibit_close)
+ close (fd);
+ xfree (fm->content);
+ xfree (fm);
+ return NULL;
}
+/* Release the resources held by FM. Specifically, this calls
+ munmap() or xfree() on fm->content, depending whether mmap or
+ malloc/read were used to read in the file. It also frees the
+ memory needed to hold the FM structure itself. */
+
+void
+read_file_free (struct file_memory *fm)
+{
+#ifdef HAVE_MMAP
+ if (fm->mmap_p)
+ {
+ munmap (fm->content, fm->length);
+ }
+ else
+#endif
+ {
+ xfree (fm->content);
+ }
+ xfree (fm);
+}
+\f
/* Free the pointers in a NULL-terminated vector of pointers, then
free the pointer itself. */
void
{
char **p = vec;
while (*p)
- free (*p++);
- free (vec);
+ xfree (*p++);
+ xfree (vec);
}
}
if (!*v2)
{
/* To avoid j == 0 */
- free (v2);
+ xfree (v2);
return v1;
}
/* Count v1. */
/* Reallocate v1. */
v1 = (char **)xrealloc (v1, (i + j + 1) * sizeof (char **));
memcpy (v1 + i, v2, (j + 1) * sizeof (char *));
- free (v2);
+ xfree (v2);
return v1;
}
-/* A set of simple-minded routines to store and search for strings in
- a linked list. You may add a string to the slist, and peek whether
- it's still in there at any time later. */
+/* A set of simple-minded routines to store strings in a linked list.
+ This used to also be used for searching, but now we have hash
+ tables for that. */
+
+/* It's a shame that these simple things like linked lists and hash
+ tables (see hash.c) need to be implemented over and over again. It
+ would be nice to be able to use the routines from glib -- see
+ www.gtk.org for details. However, that would make Wget depend on
+ glib, and I want to avoid dependencies to external libraries for
+ reasons of convenience and portability (I suspect Wget is more
+ portable than anything ever written for Gnome). */
+
+/* Append an element to the list. If the list has a huge number of
+ elements, this can get slow because it has to find the list's
+ ending. If you think you have to call slist_append in a loop,
+ think about calling slist_prepend() followed by slist_nreverse(). */
-/* Add an element to the list. If flags is NOSORT, the list will not
- be sorted. */
slist *
-add_slist (slist *l, const char *s, int flags)
+slist_append (slist *l, const char *s)
{
- slist *t, *old, *beg;
- int cmp;
+ slist *newel = (slist *)xmalloc (sizeof (slist));
+ slist *beg = l;
- if (flags & NOSORT)
- {
- if (!l)
- {
- t = (slist *)xmalloc (sizeof (slist));
- t->string = xstrdup (s);
- t->next = NULL;
- return t;
- }
- beg = l;
- /* Find the last element. */
- while (l->next)
- l = l->next;
- t = (slist *)xmalloc (sizeof (slist));
- l->next = t;
- t->string = xstrdup (s);
- t->next = NULL;
- return beg;
- }
- /* Empty list or changing the first element. */
- if (!l || (cmp = strcmp (l->string, s)) > 0)
- {
- t = (slist *)xmalloc (sizeof (slist));
- t->string = xstrdup (s);
- t->next = l;
- return t;
- }
+ newel->string = xstrdup (s);
+ newel->next = NULL;
- beg = l;
- if (cmp == 0)
- return beg;
-
- /* Second two one-before-the-last element. */
+ if (!l)
+ return newel;
+ /* Find the last element. */
while (l->next)
- {
- old = l;
- l = l->next;
- cmp = strcmp (s, l->string);
- if (cmp == 0) /* no repeating in the list */
- return beg;
- else if (cmp > 0)
- continue;
- /* If the next list element is greater than s, put s between the
- current and the next list element. */
- t = (slist *)xmalloc (sizeof (slist));
- old->next = t;
- t->next = l;
- t->string = xstrdup (s);
- return beg;
- }
- t = (slist *)xmalloc (sizeof (slist));
- t->string = xstrdup (s);
- /* Insert the new element after the last element. */
- l->next = t;
- t->next = NULL;
+ l = l->next;
+ l->next = newel;
return beg;
}
-/* Is there a specific entry in the list? */
-int
-in_slist (slist *l, const char *s)
+/* Prepend S to the list. Unlike slist_append(), this is O(1). */
+
+slist *
+slist_prepend (slist *l, const char *s)
{
- int cmp;
+ slist *newel = (slist *)xmalloc (sizeof (slist));
+ newel->string = xstrdup (s);
+ newel->next = l;
+ return newel;
+}
+/* Destructively reverse L. */
+
+slist *
+slist_nreverse (slist *l)
+{
+ slist *prev = NULL;
while (l)
{
- cmp = strcmp (l->string, s);
- if (cmp == 0)
- return 1;
- else if (cmp > 0) /* the list is ordered! */
- return 0;
- l = l->next;
+ slist *next = l->next;
+ l->next = prev;
+ prev = l;
+ l = next;
}
+ return prev;
+}
+
+/* Is there a specific entry in the list? */
+int
+slist_contains (slist *l, const char *s)
+{
+ for (; l; l = l->next)
+ if (!strcmp (l->string, s))
+ return 1;
return 0;
}
/* Free the whole slist. */
void
-free_slist (slist *l)
+slist_free (slist *l)
{
- slist *n;
-
while (l)
{
- n = l->next;
- free (l->string);
- free (l);
+ slist *n = l->next;
+ xfree (l->string);
+ xfree (l);
l = n;
}
}
+\f
+/* Sometimes it's useful to create "sets" of strings, i.e. special
+ hash tables where you want to store strings as keys and merely
+ query for their existence. Here is a set of utility routines that
+ makes that transparent. */
+
+void
+string_set_add (struct hash_table *ht, const char *s)
+{
+ /* First check whether the set element already exists. If it does,
+ do nothing so that we don't have to free() the old element and
+ then strdup() a new one. */
+ if (hash_table_exists (ht, s))
+ return;
+
+ /* We use "1" as value. It provides us a useful and clear arbitrary
+ value, and it consumes no memory -- the pointers to the same
+ string "1" will be shared by all the key-value pairs in all `set'
+ hash tables. */
+ hash_table_put (ht, xstrdup (s), "1");
+}
+
+/* Synonym for hash_table_exists... */
+
+int
+string_set_exists (struct hash_table *ht, const char *s)
+{
+ return hash_table_exists (ht, s);
+}
+
+static int
+string_set_free_mapper (void *key, void *value_ignored, void *arg_ignored)
+{
+ xfree (key);
+ return 0;
+}
+
+void
+string_set_free (struct hash_table *ht)
+{
+ hash_table_map (ht, string_set_free_mapper, NULL);
+ hash_table_destroy (ht);
+}
+
+static int
+free_keys_and_values_mapper (void *key, void *value, void *arg_ignored)
+{
+ xfree (key);
+ xfree (value);
+ return 0;
+}
+
+/* Another utility function: call free() on all keys and values of HT. */
+
+void
+free_keys_and_values (struct hash_table *ht)
+{
+ hash_table_map (ht, free_keys_and_values_mapper, NULL);
+}
+
\f
/* Engine for legible and legible_long_long; this function works on
strings. */
*p = '\0';
#endif /* (SIZEOF_LONG == 4) || (SIZEOF_LONG == 8) */
}
+\f
+/* This should probably be at a better place, but it doesn't really
+ fit into html-parse.c. */
+
+/* The function returns the pointer to the malloc-ed quoted version of
+ string s. It will recognize and quote numeric and special graphic
+ entities, as per RFC1866:
+
+ `&' -> `&'
+ `<' -> `<'
+ `>' -> `>'
+ `"' -> `"'
+ SP -> ` '
+
+ No other entities are recognized or replaced. */
+char *
+html_quote_string (const char *s)
+{
+ const char *b = s;
+ char *p, *res;
+ int i;
+
+ /* Pass through the string, and count the new size. */
+ for (i = 0; *s; s++, i++)
+ {
+ if (*s == '&')
+ i += 4; /* `amp;' */
+ else if (*s == '<' || *s == '>')
+ i += 3; /* `lt;' and `gt;' */
+ else if (*s == '\"')
+ i += 5; /* `quot;' */
+ else if (*s == ' ')
+ i += 4; /* #32; */
+ }
+ res = (char *)xmalloc (i + 1);
+ s = b;
+ for (p = res; *s; s++)
+ {
+ switch (*s)
+ {
+ case '&':
+ *p++ = '&';
+ *p++ = 'a';
+ *p++ = 'm';
+ *p++ = 'p';
+ *p++ = ';';
+ break;
+ case '<': case '>':
+ *p++ = '&';
+ *p++ = (*s == '<' ? 'l' : 'g');
+ *p++ = 't';
+ *p++ = ';';
+ break;
+ case '\"':
+ *p++ = '&';
+ *p++ = 'q';
+ *p++ = 'u';
+ *p++ = 'o';
+ *p++ = 't';
+ *p++ = ';';
+ break;
+ case ' ':
+ *p++ = '&';
+ *p++ = '#';
+ *p++ = '3';
+ *p++ = '2';
+ *p++ = ';';
+ break;
+ default:
+ *p++ = *s;
+ }
+ }
+ *p = '\0';
+ return res;
+}
projects / wget / blobdiff