written some time during the Geturl 1.0 beta cycle, and was very
inefficient and buggy. It also contained some very complex code to
remember a list of parser states, because it was supposed to be
- reentrant. The idea was that several parsers would be running
- concurrently, and you'd have pass the function a unique ID string
- (for example, the URL) by which it found the relevant parser state
- and returned the next URL. Over-engineering at its best.
+ reentrant.
The second HTML parser was written for Wget 1.4 (the first version
by the name `Wget'), and was a complete rewrite. Although the new
#include "html-parse.h"
#ifdef STANDALONE
+# undef xmalloc
+# undef xrealloc
+# undef xfree
# define xmalloc malloc
# define xrealloc realloc
# define xfree free
+# undef ISSPACE
+# undef ISDIGIT
+# undef ISXDIGIT
+# undef ISALPHA
+# undef ISALNUM
+# undef TOLOWER
+# undef TOUPPER
+
# define ISSPACE(x) isspace (x)
# define ISDIGIT(x) isdigit (x)
+# define ISXDIGIT(x) isxdigit (x)
# define ISALPHA(x) isalpha (x)
# define ISALNUM(x) isalnum (x)
# define TOLOWER(x) tolower (x)
+# define TOUPPER(x) toupper (x)
#endif /* STANDALONE */
/* Pool support. A pool is a resizable chunk of memory. It is first
struct pool {
char *contents; /* pointer to the contents. */
int size; /* size of the pool. */
- int index; /* next unoccupied position in
- contents. */
-
- int alloca_p; /* whether contents was allocated
- using alloca(). */
- char *orig_contents; /* orig_contents, allocated by
- alloca(). this is used by
- POOL_FREE to restore the pool to
- the "initial" state. */
+ int tail; /* next available position index. */
+ int resized; /* whether the pool has been resized
+ using malloc. */
+
+ char *orig_contents; /* original pool contents, usually
+ stack-allocated. used by POOL_FREE
+ to restore the pool to the initial
+ state. */
int orig_size;
};
/* Initialize the pool to hold INITIAL_SIZE bytes of storage. */
-#define POOL_INIT(pool, initial_size) do { \
- (pool).size = (initial_size); \
- (pool).contents = ALLOCA_ARRAY (char, (pool).size); \
- (pool).index = 0; \
- (pool).alloca_p = 1; \
- (pool).orig_contents = (pool).contents; \
- (pool).orig_size = (pool).size; \
+#define POOL_INIT(p, initial_storage, initial_size) do { \
+ struct pool *P = (p); \
+ P->contents = (initial_storage); \
+ P->size = (initial_size); \
+ P->tail = 0; \
+ P->resized = 0; \
+ P->orig_contents = P->contents; \
+ P->orig_size = P->size; \
} while (0)
/* Grow the pool to accomodate at least SIZE new bytes. If the pool
already has room to accomodate SIZE bytes of data, this is a no-op. */
-#define POOL_GROW(pool, increase) do { \
- int PG_newsize = (pool).index + increase; \
- DO_REALLOC_FROM_ALLOCA ((pool).contents, (pool).size, PG_newsize, \
- (pool).alloca_p, char); \
-} while (0)
+#define POOL_GROW(p, increase) \
+ GROW_ARRAY ((p)->contents, (p)->size, (p)->tail + (increase), \
+ (p)->resized, char)
/* Append text in the range [beg, end) to POOL. No zero-termination
is done. */
-#define POOL_APPEND(pool, beg, end) do { \
- const char *PA_beg = beg; \
- int PA_size = end - PA_beg; \
- POOL_GROW (pool, PA_size); \
- memcpy ((pool).contents + (pool).index, PA_beg, PA_size); \
- (pool).index += PA_size; \
+#define POOL_APPEND(p, beg, end) do { \
+ const char *PA_beg = (beg); \
+ int PA_size = (end) - PA_beg; \
+ POOL_GROW (p, PA_size); \
+ memcpy ((p)->contents + (p)->tail, PA_beg, PA_size); \
+ (p)->tail += PA_size; \
} while (0)
-/* The same as the above, but with zero termination. */
+/* Append one character to the pool. Can be used to zero-terminate
+ pool strings. */
-#define POOL_APPEND_ZT(pool, beg, end) do { \
- const char *PA_beg = beg; \
- int PA_size = end - PA_beg; \
- POOL_GROW (pool, PA_size + 1); \
- memcpy ((pool).contents + (pool).index, PA_beg, PA_size); \
- (pool).contents[(pool).index + PA_size] = '\0'; \
- (pool).index += PA_size + 1; \
+#define POOL_APPEND_CHR(p, ch) do { \
+ char PAC_char = (ch); \
+ POOL_GROW (p, 1); \
+ (p)->contents[(p)->tail++] = PAC_char; \
} while (0)
/* Forget old pool contents. The allocated memory is not freed. */
-#define POOL_REWIND(pool) pool.index = 0
+#define POOL_REWIND(p) (p)->tail = 0
/* Free heap-allocated memory for contents of POOL. This calls
xfree() if the memory was allocated through malloc. It also
values. That way after POOL_FREE, the pool is fully usable, just
as if it were freshly initialized with POOL_INIT. */
-#define POOL_FREE(pool) do { \
- if (!(pool).alloca_p) \
- xfree ((pool).contents); \
- (pool).contents = (pool).orig_contents; \
- (pool).size = (pool).orig_size; \
- (pool).index = 0; \
- (pool).alloca_p = 1; \
+#define POOL_FREE(p) do { \
+ struct pool *P = p; \
+ if (P->resized) \
+ xfree (P->contents); \
+ P->contents = P->orig_contents; \
+ P->size = P->orig_size; \
+ P->tail = 0; \
+ P->resized = 0; \
} while (0)
+/* Used for small stack-allocated memory chunks that might grow. Like
+ DO_REALLOC, this macro grows BASEVAR as necessary to take
+ NEEDED_SIZE items of TYPE.
+
+ The difference is that on the first resize, it will use
+ malloc+memcpy rather than realloc. That way you can stack-allocate
+ the initial chunk, and only resort to heap allocation if you
+ stumble upon large data.
+
+ After the first resize, subsequent ones are performed with realloc,
+ just like DO_REALLOC. */
+
+#define GROW_ARRAY(basevar, sizevar, needed_size, resized, type) do { \
+ long ga_needed_size = (needed_size); \
+ long ga_newsize = (sizevar); \
+ while (ga_newsize < ga_needed_size) \
+ ga_newsize <<= 1; \
+ if (ga_newsize != (sizevar)) \
+ { \
+ if (resized) \
+ basevar = (type *)xrealloc (basevar, ga_newsize * sizeof (type)); \
+ else \
+ { \
+ void *ga_new = xmalloc (ga_newsize * sizeof (type)); \
+ memcpy (ga_new, basevar, (sizevar) * sizeof (type)); \
+ (basevar) = ga_new; \
+ resized = 1; \
+ } \
+ (sizevar) = ga_newsize; \
+ } \
+} while (0)
\f
#define AP_DOWNCASE 1
#define AP_PROCESS_ENTITIES 2
-#define AP_SKIP_BLANKS 4
+#define AP_TRIM_BLANKS 4
/* Copy the text in the range [BEG, END) to POOL, optionally
performing operations specified by FLAGS. FLAGS may be any
- combination of AP_DOWNCASE, AP_PROCESS_ENTITIES and AP_SKIP_BLANKS
+ combination of AP_DOWNCASE, AP_PROCESS_ENTITIES and AP_TRIM_BLANKS
with the following meaning:
* AP_DOWNCASE -- downcase all the letters;
the decoded string. Recognized entities are <, >, &, ",
  and the numerical entities.
- * AP_SKIP_BLANKS -- ignore blanks at the beginning and at the end
+ * AP_TRIM_BLANKS -- ignore blanks at the beginning and at the end
of text. */
+
static void
convert_and_copy (struct pool *pool, const char *beg, const char *end, int flags)
{
- int old_index = pool->index;
+ int old_tail = pool->tail;
int size;
/* First, skip blanks if required. We must do this before entities
are processed, so that blanks can still be inserted as, for
instance, ` '. */
- if (flags & AP_SKIP_BLANKS)
+ if (flags & AP_TRIM_BLANKS)
{
while (beg < end && ISSPACE (*beg))
++beg;
if (flags & AP_PROCESS_ENTITIES)
{
- /* Stack-allocate a copy of text, process entities and copy it
- to the pool. */
- char *local_copy = (char *)alloca (size + 1);
+ /* Grow the pool, then copy the text to the pool character by
+ character, processing the encountered entities as we go
+ along.
+
+ It's safe (and necessary) to grow the pool in advance because
+ processing the entities can only *shorten* the string, it can
+ never lengthen it. */
const char *from = beg;
- char *to = local_copy;
+ char *to;
+
+ POOL_GROW (pool, end - beg);
+ to = pool->contents + pool->tail;
while (from < end)
{
const char *save = from;
int remain;
- if (++from == end) goto lose;
+ if (++from == end)
+ goto lose;
remain = end - from;
+ /* Process numeric entities "&#DDD;" and "&#xHH;". */
if (*from == '#')
{
- int numeric;
+ int numeric = 0, digits = 0;
++from;
- if (from == end || !ISDIGIT (*from)) goto lose;
- for (numeric = 0; from < end && ISDIGIT (*from); from++)
- numeric = 10 * numeric + (*from) - '0';
- if (from < end && ISALPHA (*from)) goto lose;
+ if (*from == 'x')
+ {
+ ++from;
+ for (; from < end && ISXDIGIT (*from); from++, digits++)
+ numeric = (numeric << 4) + XDIGIT_TO_NUM (*from);
+ }
+ else
+ {
+ for (; from < end && ISDIGIT (*from); from++, digits++)
+ numeric = (numeric * 10) + (*from - '0');
+ }
+ if (!digits)
+ goto lose;
numeric &= 0xff;
*to++ = numeric;
}
#define FROB(x) (remain >= (sizeof (x) - 1) \
- && !memcmp (from, x, sizeof (x) - 1) \
+ && 0 == memcmp (from, x, sizeof (x) - 1) \
&& (*(from + sizeof (x) - 1) == ';' \
|| remain == sizeof (x) - 1 \
|| !ISALNUM (*(from + sizeof (x) - 1))))
*to++ = *from++;
}
}
- *to++ = '\0';
- POOL_APPEND (*pool, local_copy, to);
+ /* Verify that we haven't exceeded the original size. (It
+ shouldn't happen, hence the assert.) */
+ assert (to - (pool->contents + pool->tail) <= end - beg);
+
+ /* Make POOL's tail point to the position following the string
+ we've written. */
+ pool->tail = to - pool->contents;
+ POOL_APPEND_CHR (pool, '\0');
}
else
{
/* Just copy the text to the pool. */
- POOL_APPEND_ZT (*pool, beg, end);
+ POOL_APPEND (pool, beg, end);
+ POOL_APPEND_CHR (pool, '\0');
}
if (flags & AP_DOWNCASE)
{
- char *p = pool->contents + old_index;
+ char *p = pool->contents + old_tail;
for (; *p; p++)
*p = TOLOWER (*p);
}
/* Map MAPFUN over HTML tags in TEXT, which is SIZE characters long.
MAPFUN will be called with two arguments: pointer to an initialized
- struct taginfo, and CLOSURE.
+ struct taginfo, and MAPARG.
ALLOWED_TAG_NAMES should be a NULL-terminated array of tag names to
be processed by this function. If it is NULL, all the tags are
void
map_html_tags (const char *text, int size,
+ void (*mapfun) (struct taginfo *, void *), void *maparg,
+ int flags,
const char **allowed_tag_names,
- const char **allowed_attribute_names,
- void (*mapfun) (struct taginfo *, void *),
- void *closure)
+ const char **allowed_attribute_names)
{
+ /* storage for strings passed to MAPFUN callback; if 256 bytes is
+ too little, POOL_APPEND allocates more with malloc. */
+ char pool_initial_storage[256];
+ struct pool pool;
+
const char *p = text;
const char *end = text + size;
- int attr_pair_count = 8;
- int attr_pair_alloca_p = 1;
- struct attr_pair *pairs = ALLOCA_ARRAY (struct attr_pair, attr_pair_count);
- struct pool pool;
+ struct attr_pair attr_pair_initial_storage[8];
+ int attr_pair_size = countof (attr_pair_initial_storage);
+ int attr_pair_resized = 0;
+ struct attr_pair *pairs = attr_pair_initial_storage;
if (!size)
return;
- POOL_INIT (pool, 256);
+ POOL_INIT (&pool, pool_initial_storage, countof (pool_initial_storage));
{
int nattrs, end_tag;
int uninteresting_tag;
look_for_tag:
- POOL_REWIND (pool);
+ POOL_REWIND (&pool);
nattrs = 0;
end_tag = 0;
declaration). */
if (*p == '!')
{
- if (!opt.strict_comments
+ if (!(flags & MHT_STRICT_COMMENTS)
&& p < end + 3 && p[1] == '-' && p[2] == '-')
{
/* If strict comments are not enforced and if we know
goto look_for_tag;
attr_raw_value_end = p; /* <foo bar="baz"> */
/* ^ */
- /* The AP_SKIP_BLANKS part is not entirely correct,
- because we don't want to skip blanks for all the
- attribute values. */
- operation = AP_PROCESS_ENTITIES | AP_SKIP_BLANKS;
+ operation = AP_PROCESS_ENTITIES;
+ if (flags & MHT_TRIM_VALUES)
+ operation |= AP_TRIM_BLANKS;
}
else
{
attr_name_end))
continue;
- DO_REALLOC_FROM_ALLOCA (pairs, attr_pair_count, nattrs + 1,
- attr_pair_alloca_p, struct attr_pair);
+ GROW_ARRAY (pairs, attr_pair_size, nattrs + 1, attr_pair_resized,
+ struct attr_pair);
- pairs[nattrs].name_pool_index = pool.index;
+ pairs[nattrs].name_pool_index = pool.tail;
convert_and_copy (&pool, attr_name_begin, attr_name_end, AP_DOWNCASE);
- pairs[nattrs].value_pool_index = pool.index;
+ pairs[nattrs].value_pool_index = pool.tail;
convert_and_copy (&pool, attr_value_begin, attr_value_end, operation);
pairs[nattrs].value_raw_beginning = attr_raw_value_begin;
pairs[nattrs].value_raw_size = (attr_raw_value_end
taginfo.start_position = tag_start_position;
taginfo.end_position = p + 1;
/* Ta-dam! */
- (*mapfun) (&taginfo, closure);
+ (*mapfun) (&taginfo, maparg);
ADVANCE (p);
}
goto look_for_tag;
}
finish:
- POOL_FREE (pool);
- if (!attr_pair_alloca_p)
+ POOL_FREE (&pool);
+ if (attr_pair_resized)
xfree (pairs);
}
projects / wget / blobdiff