/* HTML parser for Wget.
- Copyright (C) 1998, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1998, 2000, 2003 Free Software Foundation, Inc.
This file is part of GNU Wget.
You should have received a copy of the GNU General Public License
along with Wget; if not, write to the Free Software
-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+In addition, as a special exception, the Free Software Foundation
+gives permission to link the code of its release of Wget with the
+OpenSSL project's "OpenSSL" library (or with modified versions of it
+that use the same license as the "OpenSSL" library), and distribute
+the linked executables. You must obey the GNU General Public License
+in all respects for all of the code used other than "OpenSSL". If you
+modify this file, you may extend this exception to your version of the
+file, but you are not obligated to do so. If you do not wish to do
+so, delete this exception statement from your version. */
/* The only entry point to this module is map_html_tags(), which see. */
written some time during the Geturl 1.0 beta cycle, and was very
inefficient and buggy. It also contained some very complex code to
remember a list of parser states, because it was supposed to be
- reentrant. The idea was that several parsers would be running
- concurrently, and you'd have pass the function a unique ID string
- (for example, the URL) by which it found the relevant parser state
- and returned the next URL. Over-engineering at its best.
+ reentrant.
The second HTML parser was written for Wget 1.4 (the first version
by the name `Wget'), and was a complete rewrite. Although the new
#include "html-parse.h"
#ifdef STANDALONE
+# undef xmalloc
+# undef xrealloc
+# undef xfree
# define xmalloc malloc
# define xrealloc realloc
# define xfree free
+# undef ISSPACE
+# undef ISDIGIT
+# undef ISXDIGIT
+# undef ISALPHA
+# undef ISALNUM
+# undef TOLOWER
+# undef TOUPPER
+
# define ISSPACE(x) isspace (x)
# define ISDIGIT(x) isdigit (x)
+# define ISXDIGIT(x) isxdigit (x)
# define ISALPHA(x) isalpha (x)
# define ISALNUM(x) isalnum (x)
# define TOLOWER(x) tolower (x)
+# define TOUPPER(x) toupper (x)
+
+static struct options opt;
#endif /* STANDALONE */
/* Pool support. A pool is a resizable chunk of memory. It is first
is done. */
#define POOL_APPEND(pool, beg, end) do { \
- const char *PA_beg = beg; \
- int PA_size = end - PA_beg; \
+ const char *PA_beg = (beg); \
+ int PA_size = (end) - PA_beg; \
POOL_GROW (pool, PA_size); \
memcpy ((pool).contents + (pool).index, PA_beg, PA_size); \
(pool).index += PA_size; \
} while (0)
-/* The same as the above, but with zero termination. */
+/* Append one character to the pool. Can be used to zero-terminate
+ pool strings. */
-#define POOL_APPEND_ZT(pool, beg, end) do { \
- const char *PA_beg = beg; \
- int PA_size = end - PA_beg; \
- POOL_GROW (pool, PA_size + 1); \
- memcpy ((pool).contents + (pool).index, PA_beg, PA_size); \
- (pool).contents[(pool).index + PA_size] = '\0'; \
- (pool).index += PA_size + 1; \
+#define POOL_APPEND_CHR(pool, ch) do { \
+ char PAC_char = (ch); \
+ POOL_GROW (pool, 1); \
+ (pool).contents[(pool).index++] = PAC_char; \
} while (0)
/* Forget old pool contents. The allocated memory is not freed. */
\f
#define AP_DOWNCASE 1
#define AP_PROCESS_ENTITIES 2
-#define AP_SKIP_BLANKS 4
+#define AP_TRIM_BLANKS 4
/* Copy the text in the range [BEG, END) to POOL, optionally
performing operations specified by FLAGS. FLAGS may be any
- combination of AP_DOWNCASE, AP_PROCESS_ENTITIES and AP_SKIP_BLANKS
+ combination of AP_DOWNCASE, AP_PROCESS_ENTITIES and AP_TRIM_BLANKS
with the following meaning:
* AP_DOWNCASE -- downcase all the letters;
the decoded string. Recognized entities are <, >, &, ",
  and the numerical entities.
- * AP_SKIP_BLANKS -- ignore blanks at the beginning and at the end
+ * AP_TRIM_BLANKS -- ignore blanks at the beginning and at the end
of text. */
+
static void
convert_and_copy (struct pool *pool, const char *beg, const char *end, int flags)
{
/* First, skip blanks if required. We must do this before entities
are processed, so that blanks can still be inserted as, for
instance, ` '. */
- if (flags & AP_SKIP_BLANKS)
+ if (flags & AP_TRIM_BLANKS)
{
while (beg < end && ISSPACE (*beg))
++beg;
if (flags & AP_PROCESS_ENTITIES)
{
- /* Stack-allocate a copy of text, process entities and copy it
- to the pool. */
- char *local_copy = (char *)alloca (size + 1);
+ /* Grow the pool, then copy the text to the pool character by
+ character, processing the encountered entities as we go
+ along.
+
+ It's safe (and necessary) to grow the pool in advance because
+ processing the entities can only *shorten* the string, it can
+ never lengthen it. */
+ POOL_GROW (*pool, end - beg);
const char *from = beg;
- char *to = local_copy;
+ char *to = pool->contents + pool->index;
while (from < end)
{
const char *save = from;
int remain;
- if (++from == end) goto lose;
+ if (++from == end)
+ goto lose;
remain = end - from;
+ /* Process numeric entities "&#DDD;" and "&#xHH;". */
if (*from == '#')
{
- int numeric;
+ int numeric = 0, digits = 0;
++from;
- if (from == end || !ISDIGIT (*from)) goto lose;
- for (numeric = 0; from < end && ISDIGIT (*from); from++)
- numeric = 10 * numeric + (*from) - '0';
- if (from < end && ISALPHA (*from)) goto lose;
+ if (*from == 'x')
+ {
+ ++from;
+ for (; from < end && ISXDIGIT (*from); from++, digits++)
+ numeric = (numeric << 4) + XDIGIT_TO_NUM (*from);
+ }
+ else
+ {
+ for (; from < end && ISDIGIT (*from); from++, digits++)
+ numeric = (numeric * 10) + (*from - '0');
+ }
+ if (!digits)
+ goto lose;
numeric &= 0xff;
*to++ = numeric;
}
#define FROB(x) (remain >= (sizeof (x) - 1) \
- && !memcmp (from, x, sizeof (x) - 1) \
+ && 0 == memcmp (from, x, sizeof (x) - 1) \
&& (*(from + sizeof (x) - 1) == ';' \
|| remain == sizeof (x) - 1 \
|| !ISALNUM (*(from + sizeof (x) - 1))))
*to++ = *from++;
}
}
- *to++ = '\0';
- POOL_APPEND (*pool, local_copy, to);
+ /* Verify that we haven't exceeded the original size. (It
+ shouldn't happen, hence the assert.) */
+ assert (to - (pool->contents + pool->index) <= end - beg);
+
+ /* Make POOL's tail point to the position following the string
+ we've written. */
+ pool->index = to - pool->contents;
+ POOL_APPEND_CHR (*pool, '\0');
}
else
{
/* Just copy the text to the pool. */
- POOL_APPEND_ZT (*pool, beg, end);
+ POOL_APPEND (*pool, beg, end);
+ POOL_APPEND_CHR (*pool, '\0');
}
if (flags & AP_DOWNCASE)
return 1;
}
\f
-/* RFC1866: name [of attribute or tag] consists of letters, digits,
- periods, or hyphens. We also allow _, for compatibility with
- brain-damaged generators. */
-#define NAME_CHAR_P(x) (ISALNUM (x) || (x) == '.' || (x) == '-' || (x) == '_')
-
-/* States while advancing through comments. */
-#define AC_S_DONE 0
-#define AC_S_BACKOUT 1
-#define AC_S_BANG 2
-#define AC_S_DEFAULT 3
-#define AC_S_DCLNAME 4
-#define AC_S_DASH1 5
-#define AC_S_DASH2 6
-#define AC_S_COMMENT 7
-#define AC_S_DASH3 8
-#define AC_S_DASH4 9
-#define AC_S_QUOTE1 10
-#define AC_S_IN_QUOTE 11
-#define AC_S_QUOTE2 12
+/* Originally we used to adhere to rfc 1866 here, and allowed only
+ letters, digits, periods, and hyphens as names (of tags or
+ attributes). However, this broke too many pages which used
+ proprietary or strange attributes, e.g. <img src="a.gif"
+ v:shapes="whatever">.
+
+ So now we allow any character except:
+ * whitespace
+ * 8-bit and control chars
+ * characters that clearly cannot be part of name:
+ '=', '>', '/'.
+
+ This only affects attribute and tag names; attribute values allow
+ an even greater variety of characters. */
+
+#define NAME_CHAR_P(x) ((x) > 32 && (x) < 127 \
+ && (x) != '=' && (x) != '>' && (x) != '/')
#ifdef STANDALONE
static int comment_backout_count;
#endif
-/* Advance over an SGML declaration (the <!...> forms you find in HTML
- documents). The function returns the location after the
- declaration. The reason we need this is that HTML comments are
- expressed as comments in so-called "empty declarations".
+/* Advance over an SGML declaration, such as <!DOCTYPE ...>. In
+ strict comments mode, this is used for skipping over comments as
+ well.
To recap: any SGML declaration may have comments associated with
it, e.g.
<!-- have -- -- fun -->
Whitespace is allowed between and after the comments, but not
- before the first comment.
+ before the first comment. Additionally, this function attempts to
+ handle double quotes in SGML declarations correctly. */
- Additionally, this function attempts to handle double quotes in
- SGML declarations correctly. */
static const char *
advance_declaration (const char *beg, const char *end)
{
const char *p = beg;
char quote_char = '\0'; /* shut up, gcc! */
char ch;
- int state = AC_S_BANG;
+
+ enum {
+ AC_S_DONE,
+ AC_S_BACKOUT,
+ AC_S_BANG,
+ AC_S_DEFAULT,
+ AC_S_DCLNAME,
+ AC_S_DASH1,
+ AC_S_DASH2,
+ AC_S_COMMENT,
+ AC_S_DASH3,
+ AC_S_DASH4,
+ AC_S_QUOTE1,
+ AC_S_IN_QUOTE,
+ AC_S_QUOTE2,
+ } state = AC_S_BANG;
if (beg == end)
return beg;
}
break;
case AC_S_DCLNAME:
- if (NAME_CHAR_P (ch))
- ch = *p++;
- else if (ch == '-')
+ if (ch == '-')
state = AC_S_DASH1;
+ else if (NAME_CHAR_P (ch))
+ ch = *p++;
else
state = AC_S_DEFAULT;
break;
}
return p;
}
+
+/* Find the first occurrence of the substring "-->" in [BEG, END) and
+ return the pointer to the character after the substring. If the
+ substring is not found, return NULL. */
+
+static const char *
+find_comment_end (const char *beg, const char *end)
+{
+ /* Open-coded Boyer-Moore search for "-->". Examine the third char;
+ if it's not '>' or '-', advance by three characters. Otherwise,
+ look at the preceding characters and try to find a match. */
+
+ const char *p = beg - 1;
+
+ while ((p += 3) < end)
+ switch (p[0])
+ {
+ case '>':
+ if (p[-1] == '-' && p[-2] == '-')
+ return p + 1;
+ break;
+ case '-':
+ at_dash:
+ if (p[-1] == '-')
+ {
+ at_dash_dash:
+ if (++p == end) return NULL;
+ switch (p[0])
+ {
+ case '>': return p + 1;
+ case '-': goto at_dash_dash;
+ }
+ }
+ else
+ {
+ if ((p += 2) >= end) return NULL;
+ switch (p[0])
+ {
+ case '>':
+ if (p[-1] == '-')
+ return p + 1;
+ break;
+ case '-':
+ goto at_dash;
+ }
+ }
+ }
+ return NULL;
+}
\f
/* Advance P (a char pointer), with the explicit intent of being able
to read the next character. If this is not possible, go to finish. */
declaration). */
if (*p == '!')
{
- /* This is an SGML declaration -- just skip it. */
- p = advance_declaration (p, end);
+ if (!opt.strict_comments
+ && p < end + 3 && p[1] == '-' && p[2] == '-')
+ {
+ /* If strict comments are not enforced and if we know
+ we're looking at a comment, simply look for the
+ terminating "-->". Non-strict is the default because
+ it works in other browsers and most HTML writers can't
+ be bothered with getting the comments right. */
+ const char *comment_end = find_comment_end (p + 3, end);
+ if (comment_end)
+ p = comment_end;
+ }
+ else
+ {
+ /* Either in strict comment mode or looking at a non-empty
+ declaration. Real declarations are much less likely to
+ be misused the way comments are, so advance over them
+ properly regardless of strictness. */
+ p = advance_declaration (p, end);
+ }
if (p == end)
goto finish;
goto look_for_tag;
goto look_for_tag;
attr_raw_value_end = p; /* <foo bar="baz"> */
/* ^ */
- /* The AP_SKIP_BLANKS part is not entirely correct,
- because we don't want to skip blanks for all the
- attribute values. */
- operation = AP_PROCESS_ENTITIES | AP_SKIP_BLANKS;
+ /* The AP_TRIM_BLANKS is there for buggy HTML
+ generators that generate <a href=" foo"> instead of
+ <a href="foo"> (Netscape ignores spaces as well.)
+ If you really mean space, use &32; or %20. */
+ operation = AP_PROCESS_ENTITIES | AP_TRIM_BLANKS;
}
else
{