1 /* HTML parser for Wget.
2 Copyright (C) 1998, 2000 Free Software Foundation, Inc.
4 This file is part of Wget.
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or (at
9 your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
20 /* The only entry point to this module is map_html_tags(), which see. */
24 - Allow hooks for callers to process contents outside tags. This
25 is needed to implement handling <style> and <script>. The
26 taginfo structure already carries the information about where the
27 tags are, but this is not enough, because one would also want to
28 skip the comments. (The funny thing is that for <style> and
29 <script> you *don't* want to skip comments!)
31 - Create a test suite for regression testing. */
35 This is the third HTML parser written for Wget. The first one was
36 written some time during the Geturl 1.0 beta cycle, and was very
37 inefficient and buggy. It also contained some very complex code to
38 remember a list of parser states, because it was supposed to be
39 reentrant. The idea was that several parsers would be running
40 concurrently, and you'd have pass the function a unique ID string
41 (for example, the URL) by which it found the relevant parser state
42 and returned the next URL. Over-engineering at its best.
44 The second HTML parser was written for Wget 1.4 (the first version
45 by the name `Wget'), and was a complete rewrite. Although the new
46 parser behaved much better and made no claims of reentrancy, it
47 still shared many of the fundamental flaws of the old version -- it
48 only regarded HTML in terms tag-attribute pairs, where the
49 attribute's value was a URL to be returned. Any other property of
50 HTML, such as <base href=...>, or strange way to specify a URL,
51 such as <meta http-equiv=Refresh content="0; URL=..."> had to be
52 crudely hacked in -- and the caller had to be aware of these hacks.
53 Like its predecessor, this parser did not support HTML comments.
55 After Wget 1.5.1 was released, I set out to write a third HTML
56 parser. The objectives of the new parser were to: (1) provide a
57 clean way to analyze HTML lexically, (2) separate interpretation of
58 the markup from the parsing process, (3) be as correct as possible,
59 e.g. correctly skipping comments and other SGML declarations, (4)
60 understand the most common errors in markup and skip them or be
61 relaxed towrds them, and (5) be reasonably efficient (no regexps,
62 minimum copying and minimum or no heap allocation).
64 I believe this parser meets all of the above goals. It is
65 reasonably well structured, and could be relatively easily
66 separated from Wget and used elsewhere. While some of its
67 intrinsic properties limit its value as a general-purpose HTML
68 parser, I believe that, with minimum modifications, it could serve
71 Due to time and other constraints, this parser was not integrated
72 into Wget until the version ???. */
76 The single entry point of this parser is map_html_tags(), which
77 works by calling a function you specify for each tag. The function
78 gets called with the pointer to a structure describing the tag and
81 /* To test as standalone, compile with `-DSTANDALONE -I.'. You'll
82 still need Wget headers to compile. */
97 #include "html-parse.h"
100 # define xmalloc malloc
101 # define xrealloc realloc
103 #endif /* STANDALONE */
105 /* Pool support. For efficiency, map_html_tags() stores temporary
106 string data to a single stack-allocated pool. If the pool proves
107 too small, additional memory is allocated/resized with
108 malloc()/realloc(). */
111 char *contents; /* pointer to the contents. */
112 int size; /* size of the pool. */
113 int index; /* next unoccupied position in
116 int alloca_p; /* whether contents was allocated
118 char *orig_contents; /* orig_contents, allocated by
119 alloca(). this is used by
120 POOL_FREE to restore the pool to
121 the "initial" state. */
125 /* Initialize the pool to hold INITIAL_SIZE bytes of storage. */
127 #define POOL_INIT(pool, initial_size) do { \
128 (pool).size = (initial_size); \
129 (pool).contents = ALLOCA_ARRAY (char, (pool).size); \
131 (pool).alloca_p = 1; \
132 (pool).orig_contents = (pool).contents; \
133 (pool).orig_size = (pool).size; \
136 /* Grow the pool to accomodate at least SIZE new bytes. If the pool
137 already has room to accomodate SIZE bytes of data, this is a no-op. */
139 #define POOL_GROW(pool, increase) do { \
140 int PG_newsize = (pool).index + increase; \
141 DO_REALLOC_FROM_ALLOCA ((pool).contents, (pool).size, PG_newsize, \
142 (pool).alloca_p, char); \
145 /* Append text in the range [beg, end) to POOL. No zero-termination
148 #define POOL_APPEND(pool, beg, end) do { \
149 const char *PA_beg = beg; \
150 int PA_size = end - PA_beg; \
151 POOL_GROW (pool, PA_size); \
152 memcpy ((pool).contents + (pool).index, PA_beg, PA_size); \
153 (pool).index += PA_size; \
156 /* The same as the above, but with zero termination. */
158 #define POOL_APPEND_ZT(pool, beg, end) do { \
159 const char *PA_beg = beg; \
160 int PA_size = end - PA_beg; \
161 POOL_GROW (pool, PA_size + 1); \
162 memcpy ((pool).contents + (pool).index, PA_beg, PA_size); \
163 (pool).contents[(pool).index + PA_size] = '\0'; \
164 (pool).index += PA_size + 1; \
167 /* Forget old pool contents. The allocated memory is not freed. */
168 #define POOL_REWIND(pool) pool.index = 0
170 /* Free heap-allocated memory for contents of POOL. This calls
171 xfree() if the memory was allocated through malloc. It also
172 restores `contents' and `size' to their original, pre-malloc
173 values. That way after POOL_FREE, the pool is fully usable, just
174 as if it were freshly initialized with POOL_INIT. */
176 #define POOL_FREE(pool) do { \
177 if (!(pool).alloca_p) \
178 xfree ((pool).contents); \
179 (pool).contents = (pool).orig_contents; \
180 (pool).size = (pool).orig_size; \
182 (pool).alloca_p = 1; \
186 #define AP_DOWNCASE 1
187 #define AP_PROCESS_ENTITIES 2
188 #define AP_SKIP_BLANKS 4
190 /* Copy the text in the range [BEG, END) to POOL, optionally
191 performing operations specified by FLAGS. FLAGS may be any
192 combination of AP_DOWNCASE, AP_PROCESS_ENTITIES and AP_SKIP_BLANKS
193 with the following meaning:
195 * AP_DOWNCASE -- downcase all the letters;
197 * AP_PROCESS_ENTITIES -- process the SGML entities and write out
198 the decoded string. Recognized entities are <, >, &, ",
199   and the numerical entities.
201 * AP_SKIP_BLANKS -- ignore blanks at the beginning and at the end
204 convert_and_copy (struct pool *pool, const char *beg, const char *end, int flags)
206 int old_index = pool->index;
209 /* First, skip blanks if required. We must do this before entities
210 are processed, so that blanks can still be inserted as, for
211 instance, ` '. */
212 if (flags & AP_SKIP_BLANKS)
214 while (beg < end && ISSPACE (*beg))
216 while (end > beg && ISSPACE (end[-1]))
221 if (flags & AP_PROCESS_ENTITIES)
223 /* Stack-allocate a copy of text, process entities and copy it
225 char *local_copy = (char *)alloca (size + 1);
226 const char *from = beg;
227 char *to = local_copy;
235 const char *save = from;
238 if (++from == end) goto lose;
245 if (from == end || !ISDIGIT (*from)) goto lose;
246 for (numeric = 0; from < end && ISDIGIT (*from); from++)
247 numeric = 10 * numeric + (*from) - '0';
248 if (from < end && ISALPHA (*from)) goto lose;
252 #define FROB(x) (remain >= (sizeof (x) - 1) \
253 && !memcmp (from, x, sizeof (x) - 1) \
254 && (*(from + sizeof (x) - 1) == ';' \
255 || remain == sizeof (x) - 1 \
256 || !ISALNUM (*(from + sizeof (x) - 1))))
257 else if (FROB ("lt"))
258 *to++ = '<', from += 2;
259 else if (FROB ("gt"))
260 *to++ = '>', from += 2;
261 else if (FROB ("amp"))
262 *to++ = '&', from += 3;
263 else if (FROB ("quot"))
264 *to++ = '\"', from += 4;
265 /* We don't implement the proposed "Added Latin 1"
266 entities (except for nbsp), because it is unnecessary
267 in the context of Wget, and would require hashing to
269 else if (FROB ("nbsp"))
270 *to++ = 160, from += 4;
274 /* If the entity was followed by `;', we step over the
275 `;'. Otherwise, it was followed by either a
276 non-alphanumeric or EOB, in which case we do nothing. */
277 if (from < end && *from == ';')
282 /* This was not an entity after all. Back out. */
288 POOL_APPEND (*pool, local_copy, to);
292 /* Just copy the text to the pool. */
293 POOL_APPEND_ZT (*pool, beg, end);
296 if (flags & AP_DOWNCASE)
298 char *p = pool->contents + old_index;
304 /* Check whether the contents of [POS, POS+LENGTH) match any of the
305 strings in the ARRAY. */
307 array_allowed (const char **array, const char *beg, const char *end)
309 int length = end - beg;
312 for (; *array; array++)
313 if (length >= strlen (*array)
314 && !strncasecmp (*array, beg, length))
322 /* RFC1866: name [of attribute or tag] consists of letters, digits,
323 periods, or hyphens. We also allow _, for compatibility with
324 brain-damaged generators. */
325 #define NAME_CHAR_P(x) (ISALNUM (x) || (x) == '.' || (x) == '-' || (x) == '_')
327 /* States while advancing through comments. */
329 #define AC_S_BACKOUT 1
331 #define AC_S_DEFAULT 3
332 #define AC_S_DCLNAME 4
335 #define AC_S_COMMENT 7
338 #define AC_S_QUOTE1 10
339 #define AC_S_IN_QUOTE 11
340 #define AC_S_QUOTE2 12
343 static int comment_backout_count;
346 /* Advance over an SGML declaration (the <!...> forms you find in HTML
347 documents). The function returns the location after the
348 declaration. The reason we need this is that HTML comments are
349 expressed as comments in so-called "empty declarations".
351 To recap: any SGML declaration may have comments associated with
353 <!MY-DECL -- isn't this fun? -- foo bar>
355 An HTML comment is merely an empty declaration (<!>) with a comment
357 <!-- some stuff here -->
359 Several comments may be embedded in one comment declaration:
360 <!-- have -- -- fun -->
362 Whitespace is allowed between and after the comments, but not
363 before the first comment.
365 Additionally, this function attempts to handle double quotes in
366 SGML declarations correctly. */
368 advance_declaration (const char *beg, const char *end)
371 char quote_char = '\0'; /* shut up, gcc! */
373 int state = AC_S_BANG;
379 /* It looked like a good idea to write this as a state machine, but
382 while (state != AC_S_DONE && state != AC_S_BACKOUT)
385 state = AC_S_BACKOUT;
395 state = AC_S_DEFAULT;
398 state = AC_S_BACKOUT;
420 if (NAME_CHAR_P (ch))
421 state = AC_S_DCLNAME;
423 state = AC_S_BACKOUT;
428 if (NAME_CHAR_P (ch))
433 state = AC_S_DEFAULT;
436 assert (ch == '\'' || ch == '"');
437 quote_char = ch; /* cheating -- I really don't feel like
438 introducing more different states for
439 different quote characters. */
441 state = AC_S_IN_QUOTE;
444 if (ch == quote_char)
450 assert (ch == quote_char);
452 state = AC_S_DEFAULT;
464 state = AC_S_COMMENT;
467 state = AC_S_BACKOUT;
491 state = AC_S_DEFAULT;
494 state = AC_S_COMMENT;
501 if (state == AC_S_BACKOUT)
504 ++comment_backout_count;
511 /* Advance P (a char pointer), with the explicit intent of being able
512 to read the next character. If this is not possible, go to finish. */
514 #define ADVANCE(p) do { \
520 /* Skip whitespace, if any. */
522 #define SKIP_WS(p) do { \
523 while (ISSPACE (*p)) { \
528 /* Skip non-whitespace, if any. */
530 #define SKIP_NON_WS(p) do { \
531 while (!ISSPACE (*p)) { \
537 static int tag_backout_count;
540 /* Map MAPFUN over HTML tags in TEXT, which is SIZE characters long.
541 MAPFUN will be called with two arguments: pointer to an initialized
542 struct taginfo, and CLOSURE.
544 ALLOWED_TAG_NAMES should be a NULL-terminated array of tag names to
545 be processed by this function. If it is NULL, all the tags are
546 allowed. The same goes for attributes and ALLOWED_ATTRIBUTE_NAMES.
548 (Obviously, the caller can filter out unwanted tags and attributes
549 just as well, but this is just an optimization designed to avoid
550 unnecessary copying for tags/attributes which the caller doesn't
551 want to know about. These lists are searched linearly; therefore,
552 if you're interested in a large number of tags or attributes, you'd
553 better set these to NULL and filter them out yourself with a
554 hashing process most appropriate for your application.) */
557 map_html_tags (const char *text, int size,
558 const char **allowed_tag_names,
559 const char **allowed_attribute_names,
560 void (*mapfun) (struct taginfo *, void *),
563 const char *p = text;
564 const char *end = text + size;
566 int attr_pair_count = 8;
567 int attr_pair_alloca_p = 1;
568 struct attr_pair *pairs = ALLOCA_ARRAY (struct attr_pair, attr_pair_count);
574 POOL_INIT (pool, 256);
578 const char *tag_name_begin, *tag_name_end;
579 const char *tag_start_position;
580 int uninteresting_tag;
588 /* Find beginning of tag. We use memchr() instead of the usual
589 looping with ADVANCE() for speed. */
590 p = memchr (p, '<', end - p);
594 tag_start_position = p;
597 /* Establish the type of the tag (start-tag, end-tag or
601 /* This is an SGML declaration -- just skip it. */
602 p = advance_declaration (p, end);
613 while (NAME_CHAR_P (*p))
615 if (p == tag_name_begin)
619 if (end_tag && *p != '>')
622 if (!array_allowed (allowed_tag_names, tag_name_begin, tag_name_end))
623 /* We can't just say "goto look_for_tag" here because we need
624 the loop below to properly advance over the tag's attributes. */
625 uninteresting_tag = 1;
628 uninteresting_tag = 0;
629 convert_and_copy (&pool, tag_name_begin, tag_name_end, AP_DOWNCASE);
632 /* Find the attributes. */
635 const char *attr_name_begin, *attr_name_end;
636 const char *attr_value_begin, *attr_value_end;
637 const char *attr_raw_value_begin, *attr_raw_value_end;
638 int operation = AP_DOWNCASE; /* stupid compiler. */
642 /* Check for end of tag definition. */
646 /* Establish bounds of attribute name. */
647 attr_name_begin = p; /* <foo bar ...> */
649 while (NAME_CHAR_P (*p))
651 attr_name_end = p; /* <foo bar ...> */
653 if (attr_name_begin == attr_name_end)
656 /* Establish bounds of attribute value. */
658 if (NAME_CHAR_P (*p) || *p == '>')
660 /* Minimized attribute syntax allows `=' to be omitted.
661 For example, <UL COMPACT> is a valid shorthand for <UL
662 COMPACT="compact">. Even if such attributes are not
663 useful to Wget, we need to support them, so that the
664 tags containing them can be parsed correctly. */
665 attr_raw_value_begin = attr_value_begin = attr_name_begin;
666 attr_raw_value_end = attr_value_end = attr_name_end;
672 if (*p == '\"' || *p == '\'')
674 int newline_seen = 0;
675 char quote_char = *p;
676 attr_raw_value_begin = p;
678 attr_value_begin = p; /* <foo bar="baz"> */
680 while (*p != quote_char)
682 if (!newline_seen && *p == '\n')
684 /* If a newline is seen within the quotes, it
685 is most likely that someone forgot to close
686 the quote. In that case, we back out to
687 the value beginning, and terminate the tag
688 at either `>' or the delimiter, whichever
689 comes first. Such a tag terminated at `>'
691 p = attr_value_begin;
695 else if (newline_seen && *p == '>')
699 attr_value_end = p; /* <foo bar="baz"> */
701 if (*p == quote_char)
705 attr_raw_value_end = p; /* <foo bar="baz"> */
707 /* The AP_SKIP_BLANKS part is not entirely correct,
708 because we don't want to skip blanks for all the
710 operation = AP_PROCESS_ENTITIES | AP_SKIP_BLANKS;
714 attr_value_begin = p; /* <foo bar=baz> */
716 /* According to SGML, a name token should consist only
717 of alphanumerics, . and -. However, this is often
718 violated by, for instance, `%' in `width=75%'.
719 We'll be liberal and allow just about anything as
720 an attribute value. */
721 while (!ISSPACE (*p) && *p != '>')
723 attr_value_end = p; /* <foo bar=baz qux=quix> */
725 if (attr_value_begin == attr_value_end)
729 attr_raw_value_begin = attr_value_begin;
730 attr_raw_value_end = attr_value_end;
731 operation = AP_PROCESS_ENTITIES;
736 /* We skipped the whitespace and found something that is
737 neither `=' nor the beginning of the next attribute's
739 goto backout_tag; /* <foo bar /... */
743 /* If we're not interested in the tag, don't bother with any
744 of the attributes. */
745 if (uninteresting_tag)
748 /* If we aren't interested in the attribute, skip it. We
749 cannot do this test any sooner, because our text pointer
750 needs to correctly advance over the attribute. */
751 if (allowed_attribute_names
752 && !array_allowed (allowed_attribute_names, attr_name_begin,
756 DO_REALLOC_FROM_ALLOCA (pairs, attr_pair_count, nattrs + 1,
757 attr_pair_alloca_p, struct attr_pair);
759 pairs[nattrs].name_pool_index = pool.index;
760 convert_and_copy (&pool, attr_name_begin, attr_name_end, AP_DOWNCASE);
762 pairs[nattrs].value_pool_index = pool.index;
763 convert_and_copy (&pool, attr_value_begin, attr_value_end, operation);
764 pairs[nattrs].value_raw_beginning = attr_raw_value_begin;
765 pairs[nattrs].value_raw_size = (attr_raw_value_end
766 - attr_raw_value_begin);
770 if (uninteresting_tag)
776 /* By now, we have a valid tag with a name and zero or more
777 attributes. Fill in the data and call the mapper function. */
780 struct taginfo taginfo;
782 taginfo.name = pool.contents;
783 taginfo.end_tag_p = end_tag;
784 taginfo.nattrs = nattrs;
785 /* We fill in the char pointers only now, when pool can no
786 longer get realloc'ed. If we did that above, we could get
787 hosed by reallocation. Obviously, after this point, the pool
788 may no longer be grown. */
789 for (i = 0; i < nattrs; i++)
791 pairs[i].name = pool.contents + pairs[i].name_pool_index;
792 pairs[i].value = pool.contents + pairs[i].value_pool_index;
794 taginfo.attrs = pairs;
795 taginfo.start_position = tag_start_position;
796 taginfo.end_position = p + 1;
798 (*mapfun) (&taginfo, closure);
807 /* The tag wasn't really a tag. Treat its contents as ordinary
809 p = tag_start_position + 1;
815 if (!attr_pair_alloca_p)
825 test_mapper (struct taginfo *taginfo, void *arg)
829 printf ("%s%s", taginfo->end_tag_p ? "/" : "", taginfo->name);
830 for (i = 0; i < taginfo->nattrs; i++)
831 printf (" %s=%s", taginfo->attrs[i].name, taginfo->attrs[i].value);
839 char *x = (char *)xmalloc (size);
844 while ((read_count = fread (x + length, 1, size - length, stdin)))
846 length += read_count;
848 x = (char *)xrealloc (x, size);
851 map_html_tags (x, length, NULL, NULL, test_mapper, &tag_counter);
852 printf ("TAGS: %d\n", tag_counter);
853 printf ("Tag backouts: %d\n", tag_backout_count);
854 printf ("Comment backouts: %d\n", comment_backout_count);
857 #endif /* STANDALONE */